From 26c6e2e5f133139a7769065815d86c155de1a6a4 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Sat, 25 Jul 2015 20:01:11 -0400 Subject: [PATCH] Converted to docker+ dockerfiles --- docker/README.md | 24 ++++++++++++ docker/bootstrap_centos_5.dockerfile | 28 +++++++++++++ docker/bootstrap_centos_6.dockerfile | 26 +++++++++++++ docker/bootstrap_centos_7.dockerfile | 26 +++++++++++++ docker/build_dockers.bsh | 8 +++- docker/centos_script.bsh | 5 ++- docker/docker+.bsh | 39 +++++++++++++++++++ docker/git-lfs-full-build_centos_5.dockerfile | 13 +++---- docker/git-lfs-full-build_centos_6.dockerfile | 13 +++---- docker/git-lfs-full-build_centos_7.dockerfile | 13 +++---- docker/git-lfs-test_debian_7.dockerfile | 6 ++- docker/git-lfs-test_debian_8.dockerfile | 6 ++- docker/git-lfs_centos_5.dockerfile | 25 +----------- docker/git-lfs_centos_6.dockerfile | 25 +----------- docker/git-lfs_centos_7.dockerfile | 25 +----------- 15 files changed, 179 insertions(+), 103 deletions(-) create mode 100644 docker/bootstrap_centos_5.dockerfile create mode 100644 docker/bootstrap_centos_6.dockerfile create mode 100644 docker/bootstrap_centos_7.dockerfile create mode 100755 docker/docker+.bsh diff --git a/docker/README.md b/docker/README.md index cb950cbd..83b5e17d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -36,6 +36,8 @@ To only run certain docker images, supply them as arguments, e.g. And only those images will be run. +#### Environment Variables #### + There are a few environment variables you can set to easily adjust the behavior of the `run_docker.bsh` script. @@ -56,6 +58,17 @@ exit. If set to 0, the docker containers will not be automatically deleted upon exit. This can be useful for a post mortem analysis (using other docker commands not covered here). Just make sure you clean up the docker containers manually. +#### Build Environment Variables #### + +These can be before calling `run_docker.bsh`, they are actuallys just being +passed to `build_docker.bsh`. + +LFS_VERSION - The version of LFS used to bootstrap the environment. This does +not need to be bumped every version (but can be). Currently set to track git-lfs.go +version. + + + ###Development with Dockers### Sometimes you don't want to just build git-lfs and destroy the container, you @@ -249,6 +262,17 @@ directory inside the docker container. Writing to /repo in the docker will cause files to end up in ./repos/{OS NAME}/{OS VERSION #}/ + +Unlike standard Dockerfiles, these support two extra features. The first one is +the command `SOURCE`. Similar to `FROM`, only instead of inheriting the image, +it includes all the commands from another Dockerfile (Minus other `FROM` and +`MAINTAINER` commands) instead. This is useful to make multiple images that work +off of each other without having to know the container image names, and without +manually making multiple Dockerfiles have the exact same commands. + +The second feature is a variable substitution in the form of `[{ENV_VAR_NAME}]` +These will be replaced with values from calling environment or blanked out if +the environment variable is not defined. ## Docker Cheat sheet ## diff --git a/docker/bootstrap_centos_5.dockerfile b/docker/bootstrap_centos_5.dockerfile new file mode 100644 index 00000000..b1192d21 --- /dev/null +++ b/docker/bootstrap_centos_5.dockerfile @@ -0,0 +1,28 @@ +FROM centos:5 +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" + +RUN yum install -y epel-release && \ + yum install -y createrepo rsync git gnupg gnupg2 expect + + +#The purpose of this is to build and install everything needed to build git-lfs +#Next time. So that the LONG build/installed in centos are only done once, and +#stored in the image. + +#Set to master if you want the lastest, but IF there is a failure, +#the docker will not build, so I decided to make a stable version the default +ENV DOCKER_LFS_BUILD_VERSION=[{LFS_VERSION}] + +ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ +RUN cd /tmp/docker_setup/; \ + tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ + cd /tmp/docker_setup/git-lfs-*/rpm; \ + touch build.log; \ + tail -f build.log & ./build_rpms.bsh; \ + pkill tail; \ + rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* +#TODO: REMOVE this line after docker-scripts is merged into a release +RUN yum install -y --nogpgcheck /tmp/docker_setup/git-lfs-*/rpm/RPMS/noarch/golang-pkg-linux-386-1*.noarch.rpm \ No newline at end of file diff --git a/docker/bootstrap_centos_6.dockerfile b/docker/bootstrap_centos_6.dockerfile new file mode 100644 index 00000000..c33870ba --- /dev/null +++ b/docker/bootstrap_centos_6.dockerfile @@ -0,0 +1,26 @@ +FROM centos:6 +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" + +RUN yum install -y epel-release && \ + yum install -y createrepo rsync golang tar gnupg2 expect \ + golang-pkg-linux-386.noarch + +#The purpose of this is to build and install everything needed to build git-lfs +#Next time. So that the LONG build/installed in centos are only done once, and +#stored in the image. + +#Set to master if you want the lastest, but IF there is a failure, +#the docker will not build, so I decided to make a stable version the default +ENV DOCKER_LFS_BUILD_VERSION=[{LFS_VERSION}] + +ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ +RUN cd /tmp/docker_setup/; \ + tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ + cd /tmp/docker_setup/git-lfs-*/rpm; \ + touch build.log; \ + tail -f build.log & ./build_rpms.bsh; \ + pkill tail; \ + rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* diff --git a/docker/bootstrap_centos_7.dockerfile b/docker/bootstrap_centos_7.dockerfile new file mode 100644 index 00000000..06d901e1 --- /dev/null +++ b/docker/bootstrap_centos_7.dockerfile @@ -0,0 +1,26 @@ +FROM centos:7 +MAINTAINER Andy Neff + +#Docker RUN example, pass in the git-lfs checkout copy you are working with +LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" + +RUN yum install -y createrepo rsync git ruby ruby-devel golang \ + gnupg2 rpm-sign expect \ + golang-pkg-linux-386.noarch + +#The purpose of this is to build and install everything needed to build git-lfs +#Next time. So that the LONG build/installed in centos are only done once, and +#stored in the image. + +#Set to master if you want the lastest, but IF there is a failure, +#the docker will not build, so I decided to make a stable version the default +ENV DOCKER_LFS_BUILD_VERSION=[{LFS_VERSION}] + +ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ +RUN cd /tmp/docker_setup/; \ + tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ + cd /tmp/docker_setup/git-lfs-*/rpm; \ + touch build.log; \ + tail -f build.log & ./build_rpms.bsh; \ + pkill tail; \ + rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* diff --git a/docker/build_dockers.bsh b/docker/build_dockers.bsh index 9f2a3ab1..16b9861e 100755 --- a/docker/build_dockers.bsh +++ b/docker/build_dockers.bsh @@ -11,6 +11,10 @@ CUR_DIR=$(dirname "${BASH_SOURCE[0]}") #If you are not in docker group and you have sudo, default value is sudo : ${SUDO=`if ( [ ! -w /var/run/docker.sock ] && id -nG | grep -qwv docker && [ "${DOCKER_HOST:+dh}" != "dh" ] ) && which sudo > /dev/null 2>&1; then echo sudo; fi`} +if [ "${LFS_VERSION:-}" == "" ]; then + export LFS_VERSION=v$(grep '^const Version' "${CUR_DIR}/../lfs/lfs.go" | sed 's|const Version = "\([0-9.]*\)"|\1|') +fi + if [[ $# == 0 ]]; then IMAGE_NAMES=($(ls -d ${CUR_DIR}/git-lfs_*.dockerfile)) else @@ -24,6 +28,8 @@ fi #This will take a long time the first time for IMAGE_NAME in "${IMAGE_NAMES[@]}"; do echo Docker building ${IMAGE_NAME} - $SUDO docker build -t git-lfs/$(basename ${IMAGE_NAME%.*}) -f ${IMAGE_NAME} ${CUR_DIR} + ${CUR_DIR}/docker+.bsh ${IMAGE_NAME} > ${CUR_DIR}/tmp.dockerfile + $SUDO docker build -t git-lfs/$(basename ${IMAGE_NAME%.*}) -f ${CUR_DIR}/tmp.dockerfile ${CUR_DIR} + rm ${CUR_DIR}/tmp.dockerfile done diff --git a/docker/centos_script.bsh b/docker/centos_script.bsh index 7a22046a..105507c4 100755 --- a/docker/centos_script.bsh +++ b/docker/centos_script.bsh @@ -20,11 +20,12 @@ fi rpmbuild --define "_topdir ${GIT_LFS_BUILD_DIR}/rpm" -ba ${GIT_LFS_BUILD_DIR}/rpm/SPECS/git-lfs-repo-release.spec +rsync -ra ${GIT_LFS_BUILD_DIR}/rpm/{SRPMS,RPMS} ${REPO_DIR} + if [ -s /tmp/signing.key ]; then gpg --allow-secret-key-import --import /tmp/signing.key || : - find ${GIT_LFS_BUILD_DIR}/rpm/{RPMS,SRPMS} -name \*.rpm | xargs /tmp/rpm_sign.exp + find ${REPO_DIR} -name \*.rpm | xargs /tmp/rpm_sign.exp fi -rsync -ra ${GIT_LFS_BUILD_DIR}/rpm/{SRPMS,RPMS} ${REPO_DIR} createrepo ${REPO_DIR}/SRPMS createrepo ${REPO_DIR}/RPMS diff --git a/docker/docker+.bsh b/docker/docker+.bsh new file mode 100755 index 00000000..8567ff50 --- /dev/null +++ b/docker/docker+.bsh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu + +if [[ $# == 0 ]]; then + FILES=(-) +else + FILES=("${@}") +fi + +CUR_DIR=$(basename ${BASH_SOURCE[0]}) + +for FILE in "${FILES[@]}"; do + if [ "${FILE}" == "-" ]; then + FILE=/dev/stdin + fi + while IFS='' read -r LINE || [ -n "${LINE}" ]; do + + #My template substitution + while [[ "$LINE" =~ (\[\{)([A-Za-z0-9_]*)(\}\]) ]]; do + LINE=${LINE/"${BASH_REMATCH[0]}"/${!BASH_REMATCH[2]-}} + done + + #My SOURCE substitution + if echo $LINE | grep -q ^SOURCE; then + SOURCE=$(echo "${LINE}" | sed -r 's|^SOURCE\s+(\S*)\s*|\1|') + if [ "${SOURCE:0:1}" != "/" ] && [ -f "${CUR_DIR}/${SOURCE}" ]; then + SOURCE="${BASH_SOURCE[0]}" "${CUR_DIR}/${SOURCE}" + fi + if [ ! -f "${SOURCE}" ]; then + echo "Could not find a readable ${SOURCE}" >& 2 + exit 1 + fi + "${BASH_SOURCE[0]}" "${SOURCE}" | grep -vi '^FROM' | grep -vi '^MAINTAINER' + else + echo "${LINE}" + fi + done < ${FILE} +done \ No newline at end of file diff --git a/docker/git-lfs-full-build_centos_5.dockerfile b/docker/git-lfs-full-build_centos_5.dockerfile index 0b98be4c..0631c5d2 100644 --- a/docker/git-lfs-full-build_centos_5.dockerfile +++ b/docker/git-lfs-full-build_centos_5.dockerfile @@ -1,12 +1,9 @@ FROM centos:5 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" +SOURCE bootstrap_centos_5.dockerfile -RUN yum install -y createrepo rsync gnupg gnupg2 expect - -#Add the simple build repo script -COPY rpm_sign.exp signing.key centos_script.bsh /tmp/ - -CMD /tmp/centos_script.bsh \ No newline at end of file +CMD rm -rf /tmp/docker_setup/*/rpm/SRPMS/git-lfs* /tmp/docker_setup/*/rpm/RPMS/*/git-lfs* && \ + rsync -ra /tmp/docker_setup/*/rpm/{RPMS,SRPMS} /repo && \ + createrepo ${REPO_DIR}/SRPMS && \ + createrepo ${REPO_DIR}/RPMS diff --git a/docker/git-lfs-full-build_centos_6.dockerfile b/docker/git-lfs-full-build_centos_6.dockerfile index 48732cf3..b4f1fe3d 100644 --- a/docker/git-lfs-full-build_centos_6.dockerfile +++ b/docker/git-lfs-full-build_centos_6.dockerfile @@ -1,12 +1,9 @@ FROM centos:6 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" +SOURCE bootstrap_centos_6.dockerfile -RUN yum install -y createrepo rsync expect - -#Add the simple build repo script -COPY rpm_sign.exp signing.key centos_script.bsh /tmp/ - -CMD /tmp/centos_script.bsh \ No newline at end of file +CMD rm -rf /tmp/docker_setup/*/rpm/SRPMS/git-lfs* /tmp/docker_setup/*/rpm/RPMS/*/git-lfs* && \ + rsync -ra /tmp/docker_setup/*/rpm/{RPMS,SRPMS} /repo && \ + createrepo ${REPO_DIR}/SRPMS && \ + createrepo ${REPO_DIR}/RPMS diff --git a/docker/git-lfs-full-build_centos_7.dockerfile b/docker/git-lfs-full-build_centos_7.dockerfile index cd487bcb..76843dde 100644 --- a/docker/git-lfs-full-build_centos_7.dockerfile +++ b/docker/git-lfs-full-build_centos_7.dockerfile @@ -1,12 +1,9 @@ FROM centos:7 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src" -v repo_dir:/repo" +SOURCE bootstrap_centos_7.dockerfile -RUN yum install -y createrepo rsync rpm-sign expect - -#Add the simple build repo script -COPY rpm_sign.exp signing.key centos_script.bsh /tmp/ - -CMD /tmp/centos_script.bsh \ No newline at end of file +CMD rm -rf /tmp/docker_setup/*/rpm/SRPMS/git-lfs* /tmp/docker_setup/*/rpm/RPMS/*/git-lfs* && \ + rsync -ra /tmp/docker_setup/*/rpm/{RPMS,SRPMS} /repo && \ + createrepo ${REPO_DIR}/SRPMS && \ + createrepo ${REPO_DIR}/RPMS diff --git a/docker/git-lfs-test_debian_7.dockerfile b/docker/git-lfs-test_debian_7.dockerfile index 37e00b99..abbd664a 100644 --- a/docker/git-lfs-test_debian_7.dockerfile +++ b/docker/git-lfs-test_debian_7.dockerfile @@ -9,9 +9,11 @@ COPY test_lfs.bsh /tmp/test_lfs.bsh #TODO: Needs to be replaced by an apt repo COPY public.key /etc/apt/trusted.gpg.d/git-lfs.gpg COPY git-lfs-main_7.list /etc/apt/sources.list.d/git-lfs-main.list -RUN gpg --dearmour -o /etc/apt/trusted.gpg.d/stupid-debian.gpg /etc/apt/trusted.gpg.d/git-lfs.gpg +RUN [ ! -s /etc/apt/trusted.gpg.d/git-lfs.gpg ] || \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/stupid-debian.gpg /etc/apt/trusted.gpg.d/git-lfs.gpg RUN rm /etc/apt/trusted.gpg.d/git-lfs.gpg -RUN apt-key add /etc/apt/trusted.gpg.d/stupid-debian.gpg +RUN [ ! -s /etc/apt/trusted.gpg.d/stupid-debian.gpg ] || \ + apt-key add /etc/apt/trusted.gpg.d/stupid-debian.gpg #ssgelm said I didn't need to do this, but I don't know how not to. #These SHOULD be throw away commands, and not stored as Docker commits diff --git a/docker/git-lfs-test_debian_8.dockerfile b/docker/git-lfs-test_debian_8.dockerfile index 849b2e18..e2704637 100644 --- a/docker/git-lfs-test_debian_8.dockerfile +++ b/docker/git-lfs-test_debian_8.dockerfile @@ -9,9 +9,11 @@ COPY test_lfs.bsh /tmp/test_lfs.bsh #TODO: Needs to be replaced by an apt repo COPY public.key /etc/apt/trusted.gpg.d/git-lfs.gpg COPY git-lfs-main_8.list /etc/apt/sources.list.d/git-lfs-main.list -RUN gpg --dearmour -o /etc/apt/trusted.gpg.d/stupid-debian.gpg /etc/apt/trusted.gpg.d/git-lfs.gpg +RUN [ ! -s /etc/apt/trusted.gpg.d/git-lfs.gpg ] || \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/stupid-debian.gpg /etc/apt/trusted.gpg.d/git-lfs.gpg RUN rm /etc/apt/trusted.gpg.d/git-lfs.gpg -RUN apt-key add /etc/apt/trusted.gpg.d/stupid-debian.gpg +RUN [ ! -s /etc/apt/trusted.gpg.d/git-lfs.gpg ] || \ + apt-key add /etc/apt/trusted.gpg.d/stupid-debian.gpg #ssgelm said I didn't need to do this, but I don't know how not to. #These SHOULD be throw away commands, and not stored as Docker commits diff --git a/docker/git-lfs_centos_5.dockerfile b/docker/git-lfs_centos_5.dockerfile index 6dbb6712..94f2b03b 100644 --- a/docker/git-lfs_centos_5.dockerfile +++ b/docker/git-lfs_centos_5.dockerfile @@ -1,31 +1,8 @@ FROM centos:5 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" +SOURCE bootstrap_centos_5.dockerfile -RUN yum install -y epel-release && \ - yum install -y createrepo rsync git gnupg gnupg2 expect - - -#The purpose of this is to build and install everything needed to build git-lfs -#Next time. So that the LONG build/installed in centos are only done once, and -#stored in the image. - -#Set to master if you want the lastest, but IF there is a failure, -#the docker will not build, so I decided to make a stable version the default -ENV DOCKER_LFS_BUILD_VERSION=v0.5.3 - -ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ -RUN cd /tmp/docker_setup/; \ - tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ - cd /tmp/docker_setup/git-lfs-*/rpm; \ - touch build.log; \ - tail -f build.log & ./build_rpms.bsh; \ - pkill tail; \ - rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* -#TODO: REMOVE this line after docker-scripts is merged into a release -RUN yum install -y --nogpgcheck /tmp/docker_setup/git-lfs-*/rpm/RPMS/noarch/golang-pkg-linux-386-1*.noarch.rpm RUN rm -rf /tmp/docker_setup #Add the simple build repo script diff --git a/docker/git-lfs_centos_6.dockerfile b/docker/git-lfs_centos_6.dockerfile index 60632ed4..cdaf90e1 100644 --- a/docker/git-lfs_centos_6.dockerfile +++ b/docker/git-lfs_centos_6.dockerfile @@ -1,33 +1,10 @@ FROM centos:6 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" +SOURCE bootstrap_centos_6.dockerfile -RUN yum install -y epel-release && \ - yum install -y createrepo rsync golang tar gnupg2 expect \ - golang-pkg-linux-386.noarch - -#The purpose of this is to build and install everything needed to build git-lfs -#Next time. So that the LONG build/installed in centos are only done once, and -#stored in the image. - -#Set to master if you want the lastest, but IF there is a failure, -#the docker will not build, so I decided to make a stable version the default -ENV DOCKER_LFS_BUILD_VERSION=v0.5.3 - -ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ -RUN cd /tmp/docker_setup/; \ - tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ - cd /tmp/docker_setup/git-lfs-*/rpm; \ - touch build.log; \ - tail -f build.log & ./build_rpms.bsh; \ - pkill tail; \ - rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* RUN rm -rf /tmp/docker_setup - - #Add the simple build repo script COPY rpm_sign.exp signing.key centos_script.bsh /tmp/ diff --git a/docker/git-lfs_centos_7.dockerfile b/docker/git-lfs_centos_7.dockerfile index a9c20040..95c2267c 100644 --- a/docker/git-lfs_centos_7.dockerfile +++ b/docker/git-lfs_centos_7.dockerfile @@ -1,33 +1,10 @@ FROM centos:7 MAINTAINER Andy Neff -#Docker RUN example, pass in the git-lfs checkout copy you are working with -LABEL RUN="docker run -v git-lfs-repo-dir:/src -v repo_dir:/repo" +SOURCE bootstrap_centos_7.dockerfile -RUN yum install -y createrepo rsync git ruby ruby-devel golang \ - gnupg2 rpm-sign expect \ - golang-pkg-linux-386.noarch - -#The purpose of this is to build and install everything needed to build git-lfs -#Next time. So that the LONG build/installed in centos are only done once, and -#stored in the image. - -#Set to master if you want the lastest, but IF there is a failure, -#the docker will not build, so I decided to make a stable version the default -ENV DOCKER_LFS_BUILD_VERSION=v0.5.3 - -ADD https://github.com/github/git-lfs/archive/${DOCKER_LFS_BUILD_VERSION}.tar.gz /tmp/docker_setup/ -RUN cd /tmp/docker_setup/; \ - tar zxf ${DOCKER_LFS_BUILD_VERSION}.tar.gz; \ - cd /tmp/docker_setup/git-lfs-*/rpm; \ - touch build.log; \ - tail -f build.log & ./build_rpms.bsh; \ - pkill tail; \ - rm -rvf /tmp/docker_setup/git-lfs-*/rpm/BUILD* RUN rm -rf /tmp/docker_setup - - #Add the simple build repo script COPY rpm_sign.exp signing.key centos_script.bsh /tmp/