Dockers working for centos and debian with GPG signing

Fixed git clean to only be in build_rpms for centos path
Added tests for all distros
Added some options like GPG cache, and auto remove
Fixed rpm expect script to work always
Enabled gpg check for centos repos
Fixed golang build bug for CentOS 5

docker/README.md

@@ -1,8 +1,5 @@
 # README #
 
-Document
-DOCKER_CMD
-
 ## TL;DR version ##
 1. Run the dockers
 
@@ -77,21 +74,80 @@ Cleans the copies, so all untracked files are deleted, but uncommited changes ar
 
 ##Deploying/Building Repositories##
 
+When using ```BUILD_LOCAL=1```, all UNTRACKED files are removed during RPM 
+generation, except any stray directories containing a .git folder will not be
+cleared. This shouldn't be the case, unless you are temporarily storing another
+git repo in the git repo. This is a safty mechanism in git, so just keep in mind
+if you are producing packages.
+
 ### Setting the website URL ###
 
 ### GPG signing ###
 
-gpg --key-gen
+For private repo testing, GPG signing can be skipped. apt-get and yum can 
+install .deb/.rpm directly without gpg keys and everything will work. This 
+section is for distribution in a repo. Most if not all this functionality is 
+automatically disabled when there is no signing key present.
 
-public.key
+Or order to sign packages, you need to place the keys in the right place
 
-signing.key
+1. gpg --gen-key
 
-GPG agent ttl set to 5 hours, should be plenty to build everything.
+    1. 1 - RSA and RSA
+    2. 4096 bits
+    3. Some length of time or 0 for infinite
+    4. y for yes
+    5. Signer name (Will become part of the key and uid)
+    6. Email address (Will become part of the key and uid)
+    7. Comment (Will become part of the key)
+    8. O for Okay
+    9. Enter a very secure password, make sure you will not forget it
+    10. Generate Entropy!
+    
+2. gpg -a --export > ./docker/public.key
+
+3. gpg --export-secret-keys > ./docker/signing.key
+
+Keep in mind, signing.key must NEVER be accidentally commited to the repo. 
+
+To prevent MANY passphrase entries at random times, the gpg-agent is used to
+cache your signing key. This is done by running gpg-agent in the host, and passing
+the connection to each docker image. This will be done for you automatically by
+calling the ```./docker/preload_key.bsh``` script. This can be called manually
+before any other command just to get the pass phrase entry out of the way before
+you start running everything.
+
+GPG agent ttl set to 5 hours, should be plenty to build everything. If this is
+not good for you, set the GPG_MAX_CACHE and GPG_DEFAULT_CACHE environment variables
+(in seconds)
+
+[1] https://www.digitalocean.com/community/tutorials/how-to-use-reprepro-for-a-secure-package-repository-on-ubuntu-14-04
+[2] https://iuscommunity.org/pages/CreatingAGPGKeyandSigningRPMs.html#exporting-the-public-gpg-key
+[3] http://www.redhat.com/archives/rpm-list/2006-November/msg00105.html
+- Rpms do NOT SUPPORT subkeys. So don't try
 
 ### Testing the Repositories ###
 
-./test_dockers.bsh
+To test that all the OSes can download the rpm/debs, install, and run the tests
+again, run
+
+    ./test_dockers.bsh
+    
+(which is basically just ```./docker/run_dockers.bsh ./docker/git-lfs-test_*```)
+
+REPO_HOSTNAME can be used for BOTH ```run_dockers.bsh``` and ```test_dockers.bsh``` 
+to run a local test (on ```localhost:{Port Number}```, for example)
+
+An easy way to test the repositories, is to run host them on a webserver such as
+
+    cd ./docker/repos
+    python -m SimpleHTTPServer {Port number}
+
+or
+
+    cd ./docker/repos
+    ruby -run -ehttpd . -p{Port Number}
+
 
 ## Adding addition OSes ##
 

docker/centos_script.bsh

@@ -11,7 +11,7 @@ mkdir -p $(dirname "${GIT_LFS_BUILD_DIR}")
 cp -r -T "${SRC_DIR}" "${GIT_LFS_BUILD_DIR}"
 
 cd "${GIT_LFS_BUILD_DIR}"
-git clean -xdf . || ./rpm/clean.bsh #clean is needed when git isn't alread installed
+#git clean -xdf . || ./rpm/clean.bsh #clean is needed when git isn't already installed
 "${GIT_LFS_BUILD_DIR}"/rpm/build_rpms.bsh
 
 if [ "${REPO_HOSTNAME-}" != "" ]; then

docker/git-lfs-full-build_centos_5/Dockerfile

@@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 RUN yum install -y createrepo rsync
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+CMD /tmp/centos_script.bsh

docker/git-lfs-full-build_centos_6/Dockerfile

@@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 RUN yum install -y createrepo rsync
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+CMD /tmp/centos_script.bsh

docker/git-lfs-full-build_centos_7/Dockerfile

@@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 RUN yum install -y createrepo rsync
 
 #Add the simple build repo script
-ADD centos_script.bsh /tmp/docker_run/
+ADD centos_script.bsh /tmp/
 
-CMD /tmp/docker_run/centos_script.bsh
+CMD /tmp/centos_script.bsh

docker/git-lfs-test_centos_5/Dockerfile

@@ -0,0 +1,13 @@
+FROM centos:7
+MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
+
+#Docker RUN example, pass in the git-lfs checkout copy you are working with
+LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
+
+ADD test_lfs.bsh /tmp/test_lfs.bsh
+
+CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/5/RPMS/noarch/git-lfs-repo-release-1-1.noarch.rpm && \
+    yum install -y git-lfs && \
+    git-lfs && \
+    yum install -y perl-Digest-SHA golang && \
+    /tmp/test_lfs.bsh

docker/git-lfs-test_centos_6/Dockerfile

@@ -0,0 +1,13 @@
+FROM centos:7
+MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
+
+#Docker RUN example, pass in the git-lfs checkout copy you are working with
+LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
+
+ADD test_lfs.bsh /tmp/test_lfs.bsh
+
+CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/6/RPMS/noarch/git-lfs-repo-release-1-1.el6.noarch.rpm && \
+    yum install -y git-lfs && \
+    git-lfs && \
+    yum install -y perl-Digest-SHA golang && \
+    /tmp/test_lfs.bsh

docker/git-lfs-test_centos_7/Dockerfile

@@ -6,7 +6,7 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 ADD test_lfs.bsh /tmp/test_lfs.bsh
 
-CMD yum install -y --nogpgcheck http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/7/RPMS/noarch/git-lfs-repo-release-1-1.el7.centos.noarch.rpm && \
+CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/7/RPMS/noarch/git-lfs-repo-release-1-1.el7.centos.noarch.rpm && \
     yum install -y git-lfs && \
     git-lfs && \
     yum install -y perl-Digest-SHA golang && \

docker/git-lfs-test_debian_7/Dockerfile

@@ -0,0 +1,19 @@
+FROM debian:jessie
+MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
+
+#Docker RUN example, pass in the git-lfs checkout copy you are working with
+LABEL RUN="docker run -v git-lfs-checkout-dir:/src" 
+
+ADD test_lfs.bsh /tmp/test_lfs.bsh
+
+#TODO: Needs to be replaced by an apt repo
+ADD public.key /etc/apt/trusted.gpg.d/git-lfs.gpg
+ADD git-lfs-main_7.list /etc/apt/sources.list.d/git-lfs-main.list
+
+#These SHOULD be throw away commands, and not stored as Docker commits
+CMD DEBIAN_FRONTEND=noninteractive \
+    apt-get -y update && \
+    apt-get install -y git-lfs && \
+    git lfs && \
+    apt-get install -y golang curl && \
+    /tmp/test_lfs.bsh

docker/git-lfs_centos_5/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
 LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 RUN yum install -y epel-release
-RUN yum install -y createrepo rsync git gnupg gnupg2
+RUN yum install -y createrepo rsync git gnupg gnupg2 expect
 
 #The purpose of this is to build and install everything needed to build git-lfs
 #Next time. So that the LONG build/installed in centos are only done once, and
@@ -27,6 +27,4 @@ RUN rm -rf /tmp/docker_setup
 #Add the simple build repo script
 ADD rpm_sign.exp signing.key centos_script.bsh /tmp/
 
-RUN yum install -y expect
-
 CMD /tmp/centos_script.bsh

docker/git-lfs_centos_6/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Andy Neff <andyneff@users.noreply.github.com>
 LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo"
 
 RUN yum install -y epel-release
-RUN yum install -y createrepo rsync golang tar gnupg2
+RUN yum install -y createrepo rsync golang tar gnupg2 expect
 
 #The purpose of this is to build and install everything needed to build git-lfs
 #Next time. So that the LONG build/installed in centos are only done once, and
@@ -27,6 +27,4 @@ RUN rm -rf /tmp/docker_setup
 #Add the simple build repo script
 ADD rpm_sign.exp signing.key centos_script.bsh /tmp/
 
-RUN yum install -y expect
-
 CMD /tmp/centos_script.bsh

docker/preload_key.bsh

@@ -17,7 +17,7 @@ fi
 
 #Test the agent, if fail, start a new one
 if ! gpg-connect-agent /bye > /dev/null 2>&1; then
-  eval $(gpg-agent --daemon --default-cache-ttl=18000 --max-cache-ttl=18000 --write-env-file=${HOME}/.gnupg/gpg-agent.env)
+  eval $(gpg-agent --daemon --default-cache-ttl=${GPG_DEFAULT_CACHE:-18000} --max-cache-ttl=${GPG_MAX_CACHE:-18000} --write-env-file=${HOME}/.gnupg/gpg-agent.env)
 fi
 
 #Precache signing key

docker/rpm_sign.exp

@@ -1,8 +1,24 @@
-#!/usr/bin/expect -f
+#!/usr/bin/env bash
 
-#DO not edit this. The REAL password does not go in here. Use gpg-agent
+#################################################################################
+### DO not edit this. The REAL password does NOT go in here. Use gpg-agent!!! ###
+#################################################################################
+
+#$args and {*}$argv won't work in older versions of expect...
+#In case there are spaces in the files names.... 
+FILES=""
+for FILE in "${@}"; do
+  FILES+="${FILE/ /\\ } "
+done
+
+expect -f - <<EOF
+
+spawn rpm --addsign --define "__gpg $(which gpg2)" --define "_gpg_name foo" --define "__gpg_check_password_cmd /bin/true" --define "__gpg_sign_cmd %{__gpg} gpg --batch --no-verbose --no-armor --use-agent --no-secmem-warning -sbo %{__signature_filename} %{__plaintext_filename}" $FILES
 
-spawn rpm --addsign --define "_gpg_name foo" --define "__gpg_check_password_cmd /bin/true" --define "__gpg_sign_cmd %{__gpg} gpg --batch --no-verbose --no-armor --use-agent --no-secmem-warning -sbo %{__signature_filename} %{__plaintext_filename}" {*}$argv
 expect -exact "Enter pass phrase: "
+
 send -- "blank\r"
+
 expect eof
+
+EOF

docker/run_dockers.bsh

@@ -14,6 +14,10 @@
 #                checkout version
 #  DOCKER_AUTOBUILD - Default 1. If set to 0, it will not build docker images 
 #                before running
+#  AUTO_REMOVE - Default 1. If set to 0, it will not automatically delete the
+#                docker instance when done. This can be useful for a post mortem
+#                analysis. Just make sure you clean up the docker instances 
+#                manually
 
 set -eu
 
@@ -75,7 +79,11 @@ for DOCKER_FILE in "${IMAGES[@]}"; do
   fi
 
   #It CAN'T be empty () with set -u...
-  OTHER_OPTIONS=("--rm" "-it" "-e" "BUILD_LOCAL=${BUILD_LOCAL-1}")
+  OTHER_OPTIONS=("-it" "-e" "BUILD_LOCAL=${BUILD_LOCAL-1}")
+  
+  if [ "${AUTO_REMOVE-1}" == "1" ]; then
+    OTHER_OPTIONS+=("--rm")
+  fi
 
   if [ -s ${CUR_DIR}/signing.key ]; then
     set +e

rpm/SOURCES/git-lfs.repo

@@ -2,14 +2,12 @@
 name=Packages for git-lfs for Enterprise Linux $releasever - $basearch
 baseurl=http://git-lfs.github.com/centos/$releasever/RPMS
 enabled=1
-gpgcheck=0
+gpgcheck=1
-#gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-GITLFS
 
 [git-lfs-source]
 name=Packages for git-lfs for Enterprise Linux $releasever - $basearch
 baseurl=http://git-lfs.github.com/centos/$releasever/SRPMS
 enabled=0
-gpgcheck=0
+gpgcheck=1
-#gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-GITLFS

rpm/build_rpms.bsh

@@ -69,11 +69,14 @@ if [ "${BUILD_LOCAL:=0}" == "1" ]; then
   rm -rvf ${CURDIR}/tmptar
   mkdir -p ${CURDIR}/tmptar/git-lfs-${VERSION}
   pushd ${CURDIR}/..
-  tar -c . --exclude tmptar | tar -x -C ${CURDIR}/tmptar/git-lfs-${VERSION}/
+    #I started running out of space in the docker, so I needed to copy a little less waste
+    tar -c . --exclude tmptar --exclude repos | tar -x -C ${CURDIR}/tmptar/git-lfs-${VERSION}/
+  popd
+  pushd ${CURDIR}/tmptar/git-lfs-${VERSION}
+    git clean -xdf
+    rm -rvf .git
   popd
   pushd ${CURDIR}/tmptar
-    git clean -xdf
-    rm -rvf ${CURDIR}/tmptar/.git
     tar -zcf ${CURDIR}/SOURCES/git-lfs-${VERSION}.tar.gz git-lfs-${VERSION}
   popd
   rm -rvf ${CURDIR}/tmptar

rpm/clean.bsh

@@ -2,9 +2,12 @@
 
 set -eu
 
+#Emulate the important parts of git clean -xdf for CentOS 6 full build
+
 CWD=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
 
 rm -rv ${CWD}/BUILD ${CWD}/BUILDROOT ${CWD}/RPMS ${CWD}/SRPMS || :
+
 find ${CWD}/SOURCES -not -name git-lfs.repo
 
 rm ${CWD}/SPECS/golang.spec || :

rpm/golang_patch.bsh

@@ -1,23 +1,23 @@
 #!/usr/bin/env bash
 set -eu
-
+set -xv
 cd $(dirname ${BASH_SOURCE[0]})/SOURCES
 
 #Get EPEL full list
 curl -L -O https://dl.fedoraproject.org/pub/epel/fullfilelist
 #Get latest golang src rpm
 curl -L -O https://dl.fedoraproject.org/pub/epel/$(grep '6/SRPMS/golang-[0-9].*src.rpm' fullfilelist)
-rpm2cpio *.src.rpm | cpio -diuv
+rpm2cpio golang-*.src.rpm | cpio -diuv
 #Patch the spec file to patch the build to work on CentOS 5
-sed -ri 's|(^%build)|\1\nsed -i '"'"'s:.*--build-id.*::'"'"' ./src/cmd/go/build.go|' *.spec
+sed -ri 's|(^%build)|\1\nsed -i '"'"'s:.*--build-id.*::'"'"' ./src/cmd/go/build.go|' golang*.spec
 #Make SPEC CentOS 5 compliant
-sed -ri 's|(^Name:.*)|\1\nGroup: Software|' *.spec
+sed -ri 's|(^Name:.*)|\1\nGroup: Software|' golang.spec
-sed -ri 's|(^Name:.*)|\1\nBuildRoot:      %(echo %{_topdir}/BUILDROOT/%{name}-%{version})|' *.spec
+sed -ri 's|(^Name:.*)|\1\nBuildRoot:      %(echo %{_topdir}/BUILDROOT/%{name}-%{version})|' golang.spec
-sed -ri 's|(^%package\s.*)|\1\nGroup: Software|' *.spec
+sed -ri 's|(^%package\s.*)|\1\nGroup: Software|' golang.spec
-sed -i 's|%ifarch %{ix86}|%if %_arch == i686|' *.spec
+sed -i 's|%ifarch %{ix86}|%if %_arch == i686|' golang.spec
-sed -i 's|%ifarch %{arm}|%if %_arch == armv6l|' *.spec
+sed -i 's|%ifarch %{arm}|%if %_arch == armv6l|' golang.spec
-sed -i 's|%ifarch|%if %_arch ==|' *.spec
+sed -i 's|%ifarch|%if %_arch ==|' golang.spec
 #The test WILL fail, so make the rpm not fail
-sed -ri 's;(.*run.bash.*);\1|true;' *.spec
+sed -ri 's;(.*run.bash.*);\1|true;' golang.spec
-mv *.spec ../SPECS/
+mv golang.spec ../SPECS/