First draft of GPG signing working in CentOS

docker/README.md

@@ -4,13 +4,11 @@ Document
 DOCKER_CMD
 
 ## TL;DR version ##
-1. Build the docker images
-
-        ./docker/build_dockers.bsh
-
-2. Run the dockers
+1. Run the dockers
 
         ./docker/run_dockers.bsh
+        
+2. ???
 
 3. **Enjoy** all your new package files in
 
@@ -18,16 +16,21 @@ DOCKER_CMD
 
 ##Using the Dockers##
 
-### Building Dockers ###
+### Building Dockers (Optional) ###
 
-In order to use the docker **images**, they have to be built them so that they
-are ready to be used. For OSes like Debian, this is a fairly quick process. 
+```run_dockers.bsh``` now call build_dockers ```build_dockers.bsh```, but you
+can still call the script manually to get it all out of the way once while you
+go make that tea/coffee.
+
+In order to use the docker **images**, they have to be built so that they are
+ready to be used. For OSes like Debian, this is a fairly quick process. 
 However CentOS takes considerably longer time, since it has to build go, ruby,
 or git from source, depending on the version. Fortunately, you can build the 
-docker image JUST once, and you won't have to build it again (unless something
+docker images JUST once, and you won't have to build it again (unless something
 significant changes, which should be fairly uncommon). This means all the 
-compiling, yum/apt-get is done once and saved. (This is done in CentOS by
-running the ```./rpm/rpm_build.bsh``` script and saving the image.)
+compiling, yum/apt-get/custom dependency compiling is done once and saved. 
+(This is done in CentOS by using the already existing ```./rpm/rpm_build.bsh```
+script to bootstrap the image and saving the image.)
 
 There is a script to take care of ALL of these details for you. Simply run
 
@@ -80,6 +83,12 @@ Cleans the copies, so all untracked files are deleted, but uncommited changes ar
 
 gpg --key-gen
 
+public.key
+
+signing.key
+
+GPG agent ttl set to 5 hours, should be plenty to build everything.
+
 ### Testing the Repositories ###
 
 ./test_dockers.bsh
@@ -105,3 +114,45 @@ they will end up in
 ## Docker Cheat sheet ##
 
 http://docs.docker.com/ Install -> Docker Engine -> Installation on ...
+
+* list running dockers
+
+    docker ps
+    
+* list stopped dockers
+
+    docker ps -a
+    
+* Remove all stopped dockers
+
+    docker rm $(docker ps --filter=status=exited -q)
+
+1. How much space are all these Dockers taking up?
+
+    No idea. sudo du /var/lib/docker
+
+
+# Troubleshooting #
+
+1. I started one of the script, and am trying to stop it with Ctrl+C. It is
+ignoring many Ctrl+C's
+
+    This happens a lot when calling programs like apt-get, yum, etc... From the
+    host, you can still use ps, pgrep, kill, pkill, etc... commands to kill the
+    PIDs in a docker.
+    
+2. How do I re-enter a docker after it failed/suceeded?
+
+    Dockers are immediately deleted upon exit. The best way to work in a docker
+    is to run bash. This will let you to run the main build command and then
+    continue.
+    
+3. That answer's not good enough. How do I resume a docker?
+
+    Well, first you have to remove the ```--rm``` flag. This will keep the 
+    docker around after stopping. Be careful! They multiply like rabbits. Then
+    
+        ```docker commit {container name/id} {new_name}```
+    
+    Then you can ```docker run``` that new image.
+    

docker/build_dockers.bsh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Usage: 
-#  ./build_dockers.bsh - Build all the docker images
+#  ./build_dockers.bsh - Build all the compiling docker images
 #  ./build_dockers.bsh lfs_centos_5 lfs_centos_7 - Build only CentOS 5 & 7 image
 
 set -eu
@@ -13,14 +13,19 @@ CUR_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 if [[ $# == 0 ]]; then
   IMAGE_DIRS=($(ls -d ${CUR_DIR}/git-lfs_*/Dockerfile))
-  for i in "${!IMAGE_DIRS[@]}"; do
-    IMAGE_DIRS[$i]=$(dirname ${IMAGE_DIRS[$i]})
-  done
-  unset i
 else
   IMAGE_DIRS=("${@}")
 fi
 
+#Strip the Dockerfile (When appropriate)
+for i in "${!IMAGE_DIRS[@]}"; do
+  IMAGE_NAME=$(basename ${IMAGE_DIRS[$i]})
+  if [ "${IMAGE_NAME,,}" == "dockerfile" ]; then
+    IMAGE_DIRS[$i]=$(dirname ${IMAGE_DIRS[$i]})
+  fi
+done
+unset i IMAGE_NAME
+
 if [ ! -e "${CUR_DIR}/signing.key" ]; then
   touch "${CUR_DIR}/signing.key"
 fi
@@ -31,6 +36,7 @@ for IMAGE_DIR in "${IMAGE_DIRS[@]}"; do
   $SUDO docker build -t $(basename ${IMAGE_DIR}) -f ${IMAGE_DIR}/Dockerfile ${CUR_DIR}
 done
 
+#Remove it if its size zero
 if [ ! -s "${CUR_DIR}/signing.key" ]; then
   rm "${CUR_DIR}/signing.key"
 fi

docker/centos_script.bsh

@@ -20,6 +20,11 @@ fi
 
 rpmbuild --define "_topdir ${GIT_LFS_BUILD_DIR}/rpm" -ba ${GIT_LFS_BUILD_DIR}/rpm/SPECS/git-lfs-repo-release.spec
 
+if [ -s /tmp/signing.key ]; then
+  gpg --allow-secret-key-import --import /tmp/signing.key || :
+  find ${GIT_LFS_BUILD_DIR}/rpm/{RPMS,SRPMS} -name \*.rpm | xargs /tmp/rpm_sign.exp
+fi
+
 rsync -ra ${GIT_LFS_BUILD_DIR}/rpm/{SRPMS,RPMS} ${REPO_DIR}
 createrepo ${REPO_DIR}/SRPMS
 createrepo ${REPO_DIR}/RPMS

docker/debian_script.bsh

@@ -22,13 +22,15 @@ fi
 mkdir -p "${REPO_DIR}/conf/"
 sed 's|^Codename:.*|Codename: '${REPO_CODENAME}'|' /tmp/distributions > "${REPO_DIR}/conf/distributions"
 if [ -s /tmp/signing.key ]; then
-  if [ "${GPG_AGENT_INFO-}" == "" ] && [ -e "~/.gnupg/gpg-agent-info" ]; then
-    source ~/.gnupg/gpg-agent-info
-    export GPG_AGENT_INFO
-  fi
-  if ! gpg-connect 2>/dev/null; then
-    eval $(gpg-agent --daemon --write-env-file ~/.gnupg/gpg-agent-info)
-  fi
+  #Agent in the docker isn't working half the time, so I give up for now... The new way IS better
+  #if [ "${GPG_AGENT_INFO-}" == "" ] && [ -e "~/.gnupg/gpg-agent-info" ]; then
+  #  source ~/.gnupg/gpg-agent-info
+  #  export GPG_AGENT_INFO
+  #fi
+  #if ! gpg-connect-agent /bye 2>/dev/null; then
+  #  eval $(gpg-agent --daemon --write-env-file ~/.gnupg/gpg-agent-info)
+  #fi
+  #GPG_TTY=`tty`
   
   gpg --import /tmp/signing.key || :
   echo "SignWith: yes" >> "${REPO_DIR}/conf/distributions"

docker/dpkg-package-gpg.bsh

@@ -2,7 +2,7 @@
 
 ARGS=()
 while [ "$1" != "" ]; do
-  if [ "$1" == "--local-user" ]; then
+  if [ "$1" == "--local-user" ] || [ "$1" == "-u" ]; then
     shift
   else
     ARGS+=("$1")

docker/git-lfs_centos_5/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
 LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 RUN yum install -y epel-release
-RUN yum install -y createrepo rsync git
+RUN yum install -y createrepo rsync git gnupg gnupg2
 
 #The purpose of this is to build and install everything needed to build git-lfs
 #Next time. So that the LONG build/installed in centos are only done once, and
@@ -25,6 +25,8 @@ RUN cd /tmp/docker_setup/git-lfs-*/rpm; \
 RUN rm -rf /tmp/docker_setup
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD rpm_sign.exp signing.key centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+RUN yum install -y expect
+
+CMD /tmp/centos_script.bsh

docker/git-lfs_centos_6/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
 LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 RUN yum install -y epel-release
-RUN yum install -y createrepo rsync golang tar
+RUN yum install -y createrepo rsync golang tar gnupg2
 
 #The purpose of this is to build and install everything needed to build git-lfs
 #Next time. So that the LONG build/installed in centos are only done once, and
@@ -25,6 +25,8 @@ RUN cd /tmp/docker_setup/git-lfs-*/rpm; \
 RUN rm -rf /tmp/docker_setup
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD rpm_sign.exp signing.key centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+RUN yum install -y expect
+
+CMD /tmp/centos_script.bsh

docker/git-lfs_centos_7/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
 LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 
-RUN yum install -y createrepo rsync git ruby ruby-devel golang
+RUN yum install -y createrepo rsync git ruby ruby-devel golang gnupg2 rpm-sign expect
 
 #The purpose of this is to build and install everything needed to build git-lfs
 #Next time. So that the LONG build/installed in centos are only done once, and
@@ -25,6 +25,6 @@ RUN cd /tmp/docker_setup/git-lfs-*/rpm; \
 RUN rm -rf /tmp/docker_setup
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD rpm_sign.exp signing.key centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+CMD /tmp/centos_script.bsh

docker/git-lfs_debian_7/Dockerfile

@@ -9,9 +9,9 @@ RUN echo 'deb http://http.debian.net/debian wheezy-backports main' > /etc/apt/so
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y update && \
 apt-get install -y -t wheezy-backports golang git dpkg-dev dh-golang ruby-ronn reprepro
 
-ADD debian_script.bsh /tmp/docker_run/
-ADD distributions /tmp/
-ADD signing.key /tmp/
 ADD dpkg-package-gpg.bsh /tmp/
+ADD distributions /tmp/
+ADD debian_script.bsh /tmp/docker_run/
+ADD signing.key /tmp/
 
 CMD /tmp/docker_run/debian_script.bsh

docker/git-lfs_debian_8/Dockerfile

@@ -9,9 +9,9 @@ LABEL RUN="docker run -v git-lfs-checkout-dir:/src -v repo_dir:/repo"
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y update && \
 apt-get install -y golang git dpkg-dev dh-golang ruby-ronn reprepro
 
-ADD debian_script.bsh /tmp/docker_run/
-ADD distributions /tmp/
-ADD signing.key /tmp/
 ADD dpkg-package-gpg.bsh /tmp/
+ADD distributions /tmp/
+ADD debian_script.bsh /tmp/docker_run/
+ADD signing.key /tmp/
 
 CMD /tmp/docker_run/debian_script.bsh

docker/preload_key.bsh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+#set -eu
+
+if [ "$0" == "${BASH_SOURCE[0]}" ]; then
+  echo "Please source this file, do not call it"
+  exit 1
+fi
+
+#If the env isn't set, try and load it
+if [ "${GPG_AGENT_INFO-}" == "" ] || ! gpg-connect-agent /bye > /dev/null 2>&1; then
+  if [ -e "${HOME}/.gnupg/gpg-agent.env" ]; then
+    source ${HOME}/.gnupg/gpg-agent.env
+    export GPG_AGENT_INFO
+  fi
+fi
+
+#Test the agent, if fail, start a new one
+if ! gpg-connect-agent /bye > /dev/null 2>&1; then
+  eval $(gpg-agent --daemon --default-cache-ttl=18000 --max-cache-ttl=18000 --write-env-file=${HOME}/.gnupg/gpg-agent.env)
+fi
+
+#Precache signing key
+gpg2 -o /dev/null -s /dev/null

docker/rpm_sign.exp

@@ -0,0 +1,8 @@
+#!/usr/bin/expect -f
+
+#DO not edit this. The REAL password does not go in here. Use gpg-agent
+
+spawn rpm --addsign --define "_gpg_name foo" --define "__gpg_check_password_cmd /bin/true" --define "__gpg_sign_cmd %{__gpg} gpg --batch --no-verbose --no-armor --use-agent --no-secmem-warning -sbo %{__signature_filename} %{__plaintext_filename}" {*}$argv
+expect -exact "Enter pass phrase: "
+send -- "blank\r"
+expect eof

docker/run_dockers.bsh

@@ -12,6 +12,9 @@
 #                downloaded and built against.
 #                Currently only works for RPMs. DEB always builds the current
 #                checkout version
+#  DOCKER_AUTOBUILD - Default 1. If set to 0, it will not build docker images 
+#                before running
+
 set -eu
 
 CUR_DIR=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd)
@@ -70,11 +73,21 @@ for DOCKER_FILE in "${IMAGES[@]}"; do
   if [[ ${DOCKER_AUTOBUILD-1} != 0 ]]; then
     ${CUR_DIR}/build_dockers.bsh ${DOCKER_FILE}
   fi
- 
+
+  #It CAN'T be empty () with set -u...
+  OTHER_OPTIONS=("--rm" "-it" "-e" "BUILD_LOCAL=${BUILD_LOCAL-1}")
+
+  if [ -s ${CUR_DIR}/signing.key ]; then
+    set +e
+    source ${CUR_DIR}/preload_key.bsh
+    set -e
+    OTHER_OPTIONS+=("-e" "GPG_AGENT_INFO=${GPG_AGENT_INFO}")
+    OTHER_OPTIONS+=("-v" "$(dirname ${GPG_AGENT_INFO}):$(dirname ${GPG_AGENT_INFO})")
+  fi
+
   echo Compiling LFS in docker image ${IMAGE_NAME}
   IMAGE_REPO_DIR="${PACKAGE_DIR}"/"${IMAGE_INFO[1]}"/"${IMAGE_INFO[2]}"
-  $SUDO docker run --rm ${DOCKER_CMD+-it} \
-                   -e BUILD_LOCAL=${BUILD_LOCAL-1} \
+  $SUDO docker run "${OTHER_OPTIONS[@]}" \
                    -e REPO_HOSTNAME=${REPO_HOSTNAME:-git-lfs.github.com} \
                    -v ${REPO_DIR}:/src \
                    -v ${IMAGE_REPO_DIR}:/repo \