From 7ca335be0490f02f893af7435ff9e94e4d43bce5 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Mon, 20 Jul 2015 01:40:24 -0400 Subject: [PATCH] Dockers working for centos and debian with GPG signing Fixed git clean to only be in build_rpms for centos path Added tests for all distros Added some options like GPG cache, and auto remove Fixed rpm expect script to work always Enabled gpg check for centos repos Fixed golang build bug for CentOS 5 --- docker/README.md | 72 ++++++++++++++++--- docker/centos_script.bsh | 2 +- docker/git-lfs-full-build_centos_5/Dockerfile | 4 +- docker/git-lfs-full-build_centos_6/Dockerfile | 4 +- docker/git-lfs-full-build_centos_7/Dockerfile | 4 +- docker/git-lfs-test_centos_5/Dockerfile | 13 ++++ docker/git-lfs-test_centos_6/Dockerfile | 13 ++++ docker/git-lfs-test_centos_7/Dockerfile | 2 +- docker/git-lfs-test_debian_7/Dockerfile | 19 +++++ docker/git-lfs_centos_5/Dockerfile | 4 +- docker/git-lfs_centos_6/Dockerfile | 4 +- docker/preload_key.bsh | 2 +- docker/rpm_sign.exp | 24 +++++-- docker/run_dockers.bsh | 10 ++- rpm/SOURCES/git-lfs.repo | 6 +- rpm/build_rpms.bsh | 9 ++- rpm/clean.bsh | 3 + rpm/golang_patch.bsh | 22 +++--- 18 files changed, 171 insertions(+), 46 deletions(-) create mode 100644 docker/git-lfs-test_centos_5/Dockerfile create mode 100644 docker/git-lfs-test_centos_6/Dockerfile create mode 100644 docker/git-lfs-test_debian_7/Dockerfile diff --git a/docker/README.md b/docker/README.md index c638a136..dd847871 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,8 +1,5 @@ # README # -Document -DOCKER_CMD - ## TL;DR version ## 1. Run the dockers @@ -77,21 +74,80 @@ Cleans the copies, so all untracked files are deleted, but uncommited changes ar ##Deploying/Building Repositories## +When using ```BUILD_LOCAL=1```, all UNTRACKED files are removed during RPM +generation, except any stray directories containing a .git folder will not be +cleared. This shouldn't be the case, unless you are temporarily storing another +git repo in the git repo. This is a safty mechanism in git, so just keep in mind +if you are producing packages. + ### Setting the website URL ### ### GPG signing ### -gpg --key-gen +For private repo testing, GPG signing can be skipped. apt-get and yum can +install .deb/.rpm directly without gpg keys and everything will work. This +section is for distribution in a repo. Most if not all this functionality is +automatically disabled when there is no signing key present. -public.key +Or order to sign packages, you need to place the keys in the right place -signing.key +1. gpg --gen-key -GPG agent ttl set to 5 hours, should be plenty to build everything. + 1. 1 - RSA and RSA + 2. 4096 bits + 3. Some length of time or 0 for infinite + 4. y for yes + 5. Signer name (Will become part of the key and uid) + 6. Email address (Will become part of the key and uid) + 7. Comment (Will become part of the key) + 8. O for Okay + 9. Enter a very secure password, make sure you will not forget it + 10. Generate Entropy! + +2. gpg -a --export > ./docker/public.key + +3. gpg --export-secret-keys > ./docker/signing.key + +Keep in mind, signing.key must NEVER be accidentally commited to the repo. + +To prevent MANY passphrase entries at random times, the gpg-agent is used to +cache your signing key. This is done by running gpg-agent in the host, and passing +the connection to each docker image. This will be done for you automatically by +calling the ```./docker/preload_key.bsh``` script. This can be called manually +before any other command just to get the pass phrase entry out of the way before +you start running everything. + +GPG agent ttl set to 5 hours, should be plenty to build everything. If this is +not good for you, set the GPG_MAX_CACHE and GPG_DEFAULT_CACHE environment variables +(in seconds) + +[1] https://www.digitalocean.com/community/tutorials/how-to-use-reprepro-for-a-secure-package-repository-on-ubuntu-14-04 +[2] https://iuscommunity.org/pages/CreatingAGPGKeyandSigningRPMs.html#exporting-the-public-gpg-key +[3] http://www.redhat.com/archives/rpm-list/2006-November/msg00105.html +- Rpms do NOT SUPPORT subkeys. So don't try ### Testing the Repositories ### -./test_dockers.bsh +To test that all the OSes can download the rpm/debs, install, and run the tests +again, run + + ./test_dockers.bsh + +(which is basically just ```./docker/run_dockers.bsh ./docker/git-lfs-test_*```) + +REPO_HOSTNAME can be used for BOTH ```run_dockers.bsh``` and ```test_dockers.bsh``` +to run a local test (on ```localhost:{Port Number}```, for example) + +An easy way to test the repositories, is to run host them on a webserver such as + + cd ./docker/repos + python -m SimpleHTTPServer {Port number} + +or + + cd ./docker/repos + ruby -run -ehttpd . -p{Port Number} + ## Adding addition OSes ## diff --git a/docker/centos_script.bsh b/docker/centos_script.bsh index d03ac3f5..d50d683b 100755 --- a/docker/centos_script.bsh +++ b/docker/centos_script.bsh @@ -11,7 +11,7 @@ mkdir -p $(dirname "${GIT_LFS_BUILD_DIR}") cp -r -T "${SRC_DIR}" "${GIT_LFS_BUILD_DIR}" cd "${GIT_LFS_BUILD_DIR}" -git clean -xdf . || ./rpm/clean.bsh #clean is needed when git isn't alread installed +#git clean -xdf . || ./rpm/clean.bsh #clean is needed when git isn't already installed "${GIT_LFS_BUILD_DIR}"/rpm/build_rpms.bsh if [ "${REPO_HOSTNAME-}" != "" ]; then diff --git a/docker/git-lfs-full-build_centos_5/Dockerfile b/docker/git-lfs-full-build_centos_5/Dockerfile index 79a735b1..754aa04f 100644 --- a/docker/git-lfs-full-build_centos_5/Dockerfile +++ b/docker/git-lfs-full-build_centos_5/Dockerfile @@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" RUN yum install -y createrepo rsync #Add the simple build repo script -ADD centos_script.bsh /tmp/docker_run/ +ADD centos_script.bsh /tmp/ -CMD /tmp/docker_run/centos_script.bsh \ No newline at end of file +CMD /tmp/centos_script.bsh \ No newline at end of file diff --git a/docker/git-lfs-full-build_centos_6/Dockerfile b/docker/git-lfs-full-build_centos_6/Dockerfile index 3b1f29da..0336b570 100644 --- a/docker/git-lfs-full-build_centos_6/Dockerfile +++ b/docker/git-lfs-full-build_centos_6/Dockerfile @@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" RUN yum install -y createrepo rsync #Add the simple build repo script -ADD centos_script.bsh /tmp/docker_run/ +ADD centos_script.bsh /tmp/ -CMD /tmp/docker_run/centos_script.bsh \ No newline at end of file +CMD /tmp/centos_script.bsh \ No newline at end of file diff --git a/docker/git-lfs-full-build_centos_7/Dockerfile b/docker/git-lfs-full-build_centos_7/Dockerfile index 55f67cd2..3f90479e 100644 --- a/docker/git-lfs-full-build_centos_7/Dockerfile +++ b/docker/git-lfs-full-build_centos_7/Dockerfile @@ -8,6 +8,6 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" RUN yum install -y createrepo rsync #Add the simple build repo script -ADD centos_script.bsh /tmp/docker_run/ +ADD centos_script.bsh /tmp/ -CMD /tmp/docker_run/centos_script.bsh \ No newline at end of file +CMD /tmp/centos_script.bsh \ No newline at end of file diff --git a/docker/git-lfs-test_centos_5/Dockerfile b/docker/git-lfs-test_centos_5/Dockerfile new file mode 100644 index 00000000..aff99df6 --- /dev/null +++ b/docker/git-lfs-test_centos_5/Dockerfile @@ -0,0 +1,13 @@ +FROM centos:7 +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" + +ADD test_lfs.bsh /tmp/test_lfs.bsh + +CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/5/RPMS/noarch/git-lfs-repo-release-1-1.noarch.rpm && \ + yum install -y git-lfs && \ + git-lfs && \ + yum install -y perl-Digest-SHA golang && \ + /tmp/test_lfs.bsh \ No newline at end of file diff --git a/docker/git-lfs-test_centos_6/Dockerfile b/docker/git-lfs-test_centos_6/Dockerfile new file mode 100644 index 00000000..481c54d2 --- /dev/null +++ b/docker/git-lfs-test_centos_6/Dockerfile @@ -0,0 +1,13 @@ +FROM centos:7 +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" + +ADD test_lfs.bsh /tmp/test_lfs.bsh + +CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/6/RPMS/noarch/git-lfs-repo-release-1-1.el6.noarch.rpm && \ + yum install -y git-lfs && \ + git-lfs && \ + yum install -y perl-Digest-SHA golang && \ + /tmp/test_lfs.bsh \ No newline at end of file diff --git a/docker/git-lfs-test_centos_7/Dockerfile b/docker/git-lfs-test_centos_7/Dockerfile index d35b81b0..6686cf28 100644 --- a/docker/git-lfs-test_centos_7/Dockerfile +++ b/docker/git-lfs-test_centos_7/Dockerfile @@ -6,7 +6,7 @@ LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" ADD test_lfs.bsh /tmp/test_lfs.bsh -CMD yum install -y --nogpgcheck http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/7/RPMS/noarch/git-lfs-repo-release-1-1.el7.centos.noarch.rpm && \ +CMD yum install -y http://${REPO_HOSTNAME:-git-lfs.github.com}/centos/7/RPMS/noarch/git-lfs-repo-release-1-1.el7.centos.noarch.rpm && \ yum install -y git-lfs && \ git-lfs && \ yum install -y perl-Digest-SHA golang && \ diff --git a/docker/git-lfs-test_debian_7/Dockerfile b/docker/git-lfs-test_debian_7/Dockerfile new file mode 100644 index 00000000..30ec2d8e --- /dev/null +++ b/docker/git-lfs-test_debian_7/Dockerfile @@ -0,0 +1,19 @@ +FROM debian:jessie +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-checkout-dir:/src" + +ADD test_lfs.bsh /tmp/test_lfs.bsh + +#TODO: Needs to be replaced by an apt repo +ADD public.key /etc/apt/trusted.gpg.d/git-lfs.gpg +ADD git-lfs-main_7.list /etc/apt/sources.list.d/git-lfs-main.list + +#These SHOULD be throw away commands, and not stored as Docker commits +CMD DEBIAN_FRONTEND=noninteractive \ + apt-get -y update && \ + apt-get install -y git-lfs && \ + git lfs && \ + apt-get install -y golang curl && \ + /tmp/test_lfs.bsh \ No newline at end of file diff --git a/docker/git-lfs_centos_5/Dockerfile b/docker/git-lfs_centos_5/Dockerfile index a02fa666..3f258254 100644 --- a/docker/git-lfs_centos_5/Dockerfile +++ b/docker/git-lfs_centos_5/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER Andy Neff LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" RUN yum install -y epel-release -RUN yum install -y createrepo rsync git gnupg gnupg2 +RUN yum install -y createrepo rsync git gnupg gnupg2 expect #The purpose of this is to build and install everything needed to build git-lfs #Next time. So that the LONG build/installed in centos are only done once, and @@ -27,6 +27,4 @@ RUN rm -rf /tmp/docker_setup #Add the simple build repo script ADD rpm_sign.exp signing.key centos_script.bsh /tmp/ -RUN yum install -y expect - CMD /tmp/centos_script.bsh \ No newline at end of file diff --git a/docker/git-lfs_centos_6/Dockerfile b/docker/git-lfs_centos_6/Dockerfile index 343632cd..d62dc124 100644 --- a/docker/git-lfs_centos_6/Dockerfile +++ b/docker/git-lfs_centos_6/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER Andy Neff LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" RUN yum install -y epel-release -RUN yum install -y createrepo rsync golang tar gnupg2 +RUN yum install -y createrepo rsync golang tar gnupg2 expect #The purpose of this is to build and install everything needed to build git-lfs #Next time. So that the LONG build/installed in centos are only done once, and @@ -27,6 +27,4 @@ RUN rm -rf /tmp/docker_setup #Add the simple build repo script ADD rpm_sign.exp signing.key centos_script.bsh /tmp/ -RUN yum install -y expect - CMD /tmp/centos_script.bsh \ No newline at end of file diff --git a/docker/preload_key.bsh b/docker/preload_key.bsh index 6b4eec2a..67696376 100755 --- a/docker/preload_key.bsh +++ b/docker/preload_key.bsh @@ -17,7 +17,7 @@ fi #Test the agent, if fail, start a new one if ! gpg-connect-agent /bye > /dev/null 2>&1; then - eval $(gpg-agent --daemon --default-cache-ttl=18000 --max-cache-ttl=18000 --write-env-file=${HOME}/.gnupg/gpg-agent.env) + eval $(gpg-agent --daemon --default-cache-ttl=${GPG_DEFAULT_CACHE:-18000} --max-cache-ttl=${GPG_MAX_CACHE:-18000} --write-env-file=${HOME}/.gnupg/gpg-agent.env) fi #Precache signing key diff --git a/docker/rpm_sign.exp b/docker/rpm_sign.exp index de228419..b22f1e9d 100755 --- a/docker/rpm_sign.exp +++ b/docker/rpm_sign.exp @@ -1,8 +1,24 @@ -#!/usr/bin/expect -f +#!/usr/bin/env bash -#DO not edit this. The REAL password does not go in here. Use gpg-agent +################################################################################# +### DO not edit this. The REAL password does NOT go in here. Use gpg-agent!!! ### +################################################################################# + +#$args and {*}$argv won't work in older versions of expect... +#In case there are spaces in the files names.... +FILES="" +for FILE in "${@}"; do + FILES+="${FILE/ /\\ } " +done + +expect -f - <