From 24a264fce8d01c184f8dd7a123d619c0aba39e22 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 2 Apr 2020 15:09:47 -0400 Subject: [PATCH] Add scripts to allow developers to replicate CI environments To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the work automatically for you. The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci worker as a docker image and setup a container just as how gitlab-ci would be before the actual compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. ``` ./reproduce_ci_env.py create rhel8 ./reproduce_ci_env.py run rhel8 ``` To compile VTK-m from the the interactive shell you would do the following: ``` > src]# cd build/ > build]# cmake --build . ``` --- .gitlab-ci.yml | 2 +- .gitlab/ci/config/initial_config.cmake | 15 +- .gitlab/ci/ctest_build.cmake | 5 +- .gitlab/ci/ctest_configure.cmake | 14 +- .gitlab/ci/ctest_test.cmake | 5 +- Utilities/CI/.gitignore | 1 + Utilities/CI/DeveloperSetup.md | 37 +++ Utilities/CI/reproduce_ci_env.py | 307 +++++++++++++++++++++++++ Utilities/CI/requirements.txt | 1 + docs/CI-README.md | 50 +++- docs/changelog/ci-script.md | 21 ++ 11 files changed, 446 insertions(+), 12 deletions(-) create mode 100644 Utilities/CI/.gitignore create mode 100644 Utilities/CI/DeveloperSetup.md create mode 100755 Utilities/CI/reproduce_ci_env.py create mode 100644 Utilities/CI/requirements.txt create mode 100644 docs/changelog/ci-script.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 732f4d5e8..79c5f8940 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,8 +105,8 @@ stages: - sccache --show-stats - "cmake --version" - "cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake" - script: - "ctest -VV -S .gitlab/ci/ctest_configure.cmake" + script: - "ctest -VV -S .gitlab/ci/ctest_build.cmake" - sccache --show-stats artifacts: diff --git a/.gitlab/ci/config/initial_config.cmake b/.gitlab/ci/config/initial_config.cmake index a4d64a537..ecb6cfa90 100644 --- a/.gitlab/ci/config/initial_config.cmake +++ b/.gitlab/ci/config/initial_config.cmake @@ -63,8 +63,15 @@ foreach(option IN LISTS options) endforeach() set(CTEST_USE_LAUNCHERS "ON" CACHE STRING "") -set(CMAKE_C_COMPILER_LAUNCHER "sccache" CACHE STRING "") -set(CMAKE_CXX_COMPILER_LAUNCHER "sccache" CACHE STRING "") -if(VTKm_ENABLE_CUDA) - set(CMAKE_CUDA_COMPILER_LAUNCHER "sccache" CACHE STRING "") + +# We need to store the absolute path so that +# the launcher still work even when sccache isn't +# on our path. +find_program(SCCACHE_COMMAND NAMES sccache) +if(SCCACHE_COMMAND) + set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") + set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") + if(VTKm_ENABLE_CUDA) + set(CMAKE_CUDA_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") + endif() endif() diff --git a/.gitlab/ci/ctest_build.cmake b/.gitlab/ci/ctest_build.cmake index b9ae3e019..6a7f012be 100644 --- a/.gitlab/ci/ctest_build.cmake +++ b/.gitlab/ci/ctest_build.cmake @@ -24,7 +24,10 @@ message(STATUS "CTEST_BUILD_FLAGS: ${CTEST_BUILD_FLAGS}") ctest_build(APPEND NUMBER_WARNINGS num_warnings RETURN_VALUE build_result) -ctest_submit(PARTS Build) + +if(NOT DEFINED ENV{GITLAB_CI_EMULATION}) + ctest_submit(PARTS Build) +endif() if (build_result) message(FATAL_ERROR diff --git a/.gitlab/ci/ctest_configure.cmake b/.gitlab/ci/ctest_configure.cmake index b1ce37b31..0e0c9d999 100644 --- a/.gitlab/ci/ctest_configure.cmake +++ b/.gitlab/ci/ctest_configure.cmake @@ -27,16 +27,22 @@ ctest_start(Experimental TRACK "${CTEST_TRACK}") find_package(Git) set(CTEST_UPDATE_VERSION_ONLY ON) set(CTEST_UPDATE_COMMAND "${GIT_EXECUTABLE}") -ctest_update() + +# Don't do updates when running via reproduce_ci_env.py +if(NOT DEFINED ENV{GITLAB_CI_EMULATION}) + ctest_update() +endif() # Configure the project. ctest_configure(APPEND OPTIONS "${cmake_args}" RETURN_VALUE configure_result) -# We can now submit because we've configured. This is a cmb-superbuild-ism. -ctest_submit(PARTS Update) -ctest_submit(PARTS Configure) +# We can now submit because we've configured. +if(NOT DEFINED ENV{GITLAB_CI_EMULATION}) + ctest_submit(PARTS Update) + ctest_submit(PARTS Configure) +endif() if (configure_result) message(FATAL_ERROR diff --git a/.gitlab/ci/ctest_test.cmake b/.gitlab/ci/ctest_test.cmake index c2d8eab51..4b7abd4c9 100644 --- a/.gitlab/ci/ctest_test.cmake +++ b/.gitlab/ci/ctest_test.cmake @@ -33,7 +33,10 @@ ctest_test(APPEND EXCLUDE "${test_exclusions}" REPEAT "UNTIL_PASS:3" ) -ctest_submit(PARTS Test) + +if(NOT DEFINED ENV{GITLAB_CI_EMULATION}) + ctest_submit(PARTS Test) +endif() if (test_result) message(FATAL_ERROR diff --git a/Utilities/CI/.gitignore b/Utilities/CI/.gitignore new file mode 100644 index 000000000..bdaab25d5 --- /dev/null +++ b/Utilities/CI/.gitignore @@ -0,0 +1 @@ +env/ diff --git a/Utilities/CI/DeveloperSetup.md b/Utilities/CI/DeveloperSetup.md new file mode 100644 index 000000000..22196e6f0 --- /dev/null +++ b/Utilities/CI/DeveloperSetup.md @@ -0,0 +1,37 @@ +#How to setup machine to use CI scripts# + +#OSX and Unix# + + +# Requirements # + +- Docker +- Python3 +-- PyYAML + +The CI scripts require python3 and the PyYAML package. + +Generally the best way to setup this environment is to create a python +virtual env so you don't pollute your system. This means getting pip +the python package manager, and virtual env which allow for isolation +of a projects python dependencies. + +``` +sudo easy_install pip +sudo pip install virtualenv +``` + +Next we need to create a new virtual env of python. I personally +like to setup this in `vtkm/Utilities/CI/env`. + +``` +mkdir env +virtualenv env +``` + +Now all we have to do is setup the requirements: + +``` +./env/bin/pip install -r requirements.txt +``` + diff --git a/Utilities/CI/reproduce_ci_env.py b/Utilities/CI/reproduce_ci_env.py new file mode 100755 index 000000000..62a682f6a --- /dev/null +++ b/Utilities/CI/reproduce_ci_env.py @@ -0,0 +1,307 @@ +#!/bin/env python3 + +#============================================================================= +# +# Copyright (c) Kitware, Inc. +# All rights reserved. +# See LICENSE.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even +# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the above copyright notice for more information. +# +#=============== + +import enum +import os +import tempfile +import string +import subprocess +import sys +import platform +import re +import yaml + +def get_root_dir(): + dir_path = os.path.dirname(os.path.realpath(__file__)) + #find the where .gitlab-ci.yml is located + try: + src_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=dir_path) + src_root = str(src_root, 'utf-8') + src_root = src_root.rstrip('\n') + # Corrections in case the filename is a funny Cygwin path + src_root = re.sub(r'^/cygdrive/([a-z])/', r'\1:/', src_root) + return src_root + except subprocess.CalledProcessError: + return None + +def extract_stage_job_from_cmdline(*args): + if len(args) == 1: + stage_and_job = str(args[0]).split(':') + if len(stage_and_job) == 1: + stage_and_job = ['build', stage_and_job[0]] + return stage_and_job + return args + +def load_ci_file(ci_file_path): + ci_state = {} + if ci_file_path: + root_dir = os.path.dirname(ci_file_path) + ci_state = yaml.safe_load(open(ci_file_path)) + if 'include' in ci_state: + for inc in ci_state['include']: + if 'local' in inc: + #the local paths can start with '/' + include_path = inc['local'].lstrip('/') + include_path = os.path.join(root_dir, include_path) + ci_state.update(yaml.safe_load(open(include_path))) + return ci_state + +def ci_stages_and_jobs(ci_state): + stages = ci_state['stages'] + jobs = dict((s,[]) for s in stages) + for key in ci_state: + maybe_stage = key.split(':') + if maybe_stage[0] in stages: + jobs[maybe_stage[0]].append(maybe_stage[1]) + return jobs + +def subset_yml(ci_state, stage, name): + #given a stage and name generate a new yaml + #file that only contains information for stage and name. + #Does basic extend merging so that recreating the env is easier + runner_yml = {} + yml_name = stage+":"+name + runner_yml[yml_name] = ci_state[yml_name] + entry = runner_yml[yml_name] + + #Flatten 'extends' entries, only presume the first level of inheritance is + #important + if 'extends' in entry: + to_merge = [] + + if not isinstance(entry['extends'], list): + entry['extends'] = [ entry['extends'] ] + + for e in entry['extends']: + entry.update(ci_state[e]) + del entry['extends'] + return runner_yml + +class CallMode(enum.Enum): + call = 1 + output = 2 + + +def subprocess_call_docker(cmd, cwd, mode=CallMode.call): + system = platform.system() + if (system == 'Windows') or (system == 'Darwin'): + # Windows and MacOS run Docker in a VM, so they don't need sudo + full_cmd = ['docker'] + cmd + else: + # Unix needs to run docker with root privileges + full_cmd = ['sudo', 'docker'] + cmd + print(" ".join(full_cmd), flush=True) + + if mode is CallMode.call: + return subprocess.check_call(full_cmd, cwd=cwd) + if mode is CallMode.output: + return subprocess.check_output(full_cmd, cwd=cwd) + +############################################################################### +# +# User Command: 'list' +# +############################################################################### +def list_jobs(ci_file_path, *args): + ci_state = load_ci_file(ci_file_path) + jobs = ci_stages_and_jobs(ci_state) + for key,values in jobs.items(): + print('Jobs for Stage:', key) + for v in values: + print('\t',v) + print('') + + +############################################################################### +# +# User Command: 'build' | 'setup' +# +############################################################################### +def create_container(ci_file_path, *args): + ci_state = load_ci_file(ci_file_path) + ci_jobs = ci_stages_and_jobs(ci_state) + stage,name = extract_stage_job_from_cmdline(*args) + + if not stage in ci_jobs: + print('Unable to find stage: ', stage) + print('Valid stages are:', list(ci_jobs.keys())) + exit(1) + + if not name in ci_jobs[stage]: + print('Unable to find job: ', name) + print('Valid jobs are:', ci_jobs[stage]) + exit(1) + + #we now have the relevant subset of the yml + #fully expanded into a single definition + subset = subset_yml(ci_state, stage, name) + + runner_name = stage+":"+name + runner = subset[runner_name] + src_dir = get_root_dir() + gitlab_env = [ k + '="' + v + '"' for k,v in runner['variables'].items()] + + # propagate any https/http proxy info + if os.getenv('http_proxy'): + gitlab_env = [ 'http_proxy=' + os.getenv('http_proxy') ] + gitlab_env + if os.getenv('https_proxy'): + gitlab_env = [ 'https_proxy=' + os.getenv('https_proxy') ] + gitlab_env + + # The script and before_script could be anywhere! + script_search_locations = [ci_state, subset, runner] + for loc in script_search_locations: + if 'before_script' in loc: + before_script = loc['before_script'] + if 'script' in loc: + script = loc['script'] + + docker_template = string.Template(''' +FROM $image +ENV GITLAB_CI=1 \ + GITLAB_CI_EMULATION=1 \ + CI_PROJECT_DIR=. \ + CI_JOB_NAME=$job_name +#Copy all of this project to the src directory +COPY . /src +ENV $gitlab_env +WORKDIR /src +#Let git fix issues from copying across OS (such as windows EOL) +#Note that this will remove any changes not committed. +RUN echo "$before_script || true" >> /setup-gitlab-env.sh && \ + echo "$script || true" >> /run-gitlab-stage.sh && \ + git reset --hard && \ + bash /setup-gitlab-env.sh +''') + + docker_content = docker_template.substitute(image=runner['image'], + job_name='local-build'+runner_name, + src_dir=src_dir, + gitlab_env= " ".join(gitlab_env), + before_script=" && ".join(before_script), + script=" && ".join(script)) + + # Write out the file + docker_file = tempfile.NamedTemporaryFile(delete=False) + docker_file.write(bytes(docker_content, 'utf-8')) + docker_file.close() + + # now we need to run docker and build this image with a name equal to the + # ci name, and the docker context to be the current git repo root dir so + # we can copy the current project src automagically + try: + subprocess_call_docker(['build', '-f', docker_file.name, '-t', runner_name, src_dir], + cwd=src_dir) + except subprocess.CalledProcessError: + print('Unable to build the docker image for: ', runner_name) + exit(1) + finally: + # remove the temp file + os.remove(docker_file.name) + +############################################################################### +# +# User Command: 'help' +# +############################################################################### +def run_container(ci_file_path, *args): + # Exec/Run ( https://docs.docker.com/engine/reference/commandline/exec/#run-docker-exec-on-a-running-container ) + src_dir = get_root_dir() + stage,name = extract_stage_job_from_cmdline(*args) + image_name = stage+':'+name + + try: + cmd = ['run', '-itd', image_name] + container_id = subprocess_call_docker(cmd, cwd=src_dir, mode=CallMode.output) + container_id = str(container_id, 'utf-8') + container_id= container_id.rstrip('\n') + except subprocess.CalledProcessError: + print('Unable to run the docker image for: ', image_name) + exit(1) + + try: + cmd = ['exec', '-it', container_id, 'bash'] + subprocess_call_docker(cmd, cwd=src_dir) + except subprocess.CalledProcessError: + print('Unable to attach an iteractive shell to : ', container_id) + pass + + try: + cmd = ['container', 'stop', container_id] + subprocess_call_docker(cmd, cwd=src_dir) + except subprocess.CalledProcessError: + print('Unable to stop container: ', container_id) + pass + +############################################################################### +# +# User Command: 'help' +# +############################################################################### +def help_usage(ci_file_path, *args): + print('Setup gitlab-ci docker environments/state locally') + print('Usage: reproduce_ci_env.py [command] [stage] ') + print('\n') + print('Commands:\n' + \ + '\n'+\ + ' list: List all stage and job names for gitlab-ci\n'+\ + ' create: build a docker container for this gitlab-ci job.\n'+\ + ' Will match the to docker repo, and to the tag. \n' +\ + ' If no explicit is provided will default to `build` stage. \n' +\ + ' run: Launch an interactive shell inside the docker image\n' +\ + ' for a given stage:name with the correct environment and will automatically\n' +\ + ' run the associated stage script.\n' + ' If no explicit is provided will default to `build` stage. \n') + print('Example:\n' + \ + '\n'+\ + ' reproduce_ci_env create centos7\n'+\ + ' reproduce_ci_env run build:centos7\n') + +############################################################################### +def main(argv): + ci_file_path = os.path.join(get_root_dir(), '.gitlab-ci.yml') + if len(argv) == 0: + help_usage( ci_file_path ) + exit(1) + if len(argv) > 3: + help_usage( ci_file_path ) + exit(1) + + #commands we want + # - list + # -- list all 'jobs' + # - create | setup + # -- create a docker image that represents a given stage:name + # - run | exec + # -- run the script for the stage:name inside the correct docker image + # and provide an interactive shell + # -- help + #setup arg function table + commands = { + 'list': list_jobs, + 'create': create_container, + 'setup': create_container, + 'exec': run_container, + 'run': run_container, + 'help': help_usage + } + if argv[0] in commands: + #splat the subset of the vector so they are separate call parameters + commands[argv[0]]( ci_file_path, *argv[1:3] ) + else: + commands['help']( ci_file_path ) + exit(1) + exit(0) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/Utilities/CI/requirements.txt b/Utilities/CI/requirements.txt new file mode 100644 index 000000000..5500f007d --- /dev/null +++ b/Utilities/CI/requirements.txt @@ -0,0 +1 @@ +PyYAML diff --git a/docs/CI-README.md b/docs/CI-README.md index da77324ed..7a05254bd 100644 --- a/docs/CI-README.md +++ b/docs/CI-README.md @@ -71,10 +71,58 @@ Current gitlab runner tags for VTK-m are: run the VTK-m tests # How to use docker builders locally -## Setting up docker + +When diagnosing issues from the docker builders it can be useful to iterate locally on a +solution. + +If you haven't set up docker locally we recommend following the official getting started guide: + - https://docs.docker.com/get-started/ + + ## Setting up nvidia runtime + +To properly test VTK-m inside docker containers when the CUDA backend is enabled you will need +to have installed the nvidia-container-runtime ( https://github.com/NVIDIA/nvidia-container-runtime ) +and be using a recent version of docker ( we recommend docker-ce ) + + +Once nvidia-container-runtime is installed you will want the default-runtime be `nvidia` so +that `docker run` will automatically support gpus. The easiest way to do so is to add +the following to your `/etc/docker/daemon.json` + +``` +{ + "default-runtime": "nvidia", + "runtimes": { + "nvidia": { + "path": "/usr/bin/nvidia-container-runtime", + "runtimeArgs": [] + } + }, +} +``` + ## Running docker images +To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the +work automatically for you. + +The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. + +To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci +worker as a docker image and setup a container just as how gitlab-ci would be before the actual +compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. + +``` +./reproduce_ci_env.py create rhel8 +./reproduce_ci_env.py run rhel8 +``` + +To compile VTK-m from the the interactive shell you would do the following: +``` +> src]# cd build/ +> build]# cmake --build . +``` # How to Add/Update Kitware Gitlab CI diff --git a/docs/changelog/ci-script.md b/docs/changelog/ci-script.md new file mode 100644 index 000000000..f3f85a1f9 --- /dev/null +++ b/docs/changelog/ci-script.md @@ -0,0 +1,21 @@ +# Provide scripts to build Gitlab-ci workers locally + +To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the +work automatically for you. + +The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. + +To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci +worker as a docker image and setup a container just as how gitlab-ci would be before the actual +compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. + +``` +./reproduce_ci_env.py create rhel8 +./reproduce_ci_env.py run rhel8 +``` + +To compile VTK-m from the the interactive shell you would do the following: +``` +> src]# cd build/ +> build]# cmake --build . +```