mirror of
https://gitlab.kitware.com/vtk/vtk-m
synced 2024-10-05 01:49:02 +00:00
Add scripts to allow developers to replicate CI environments
To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the work automatically for you. The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci worker as a docker image and setup a container just as how gitlab-ci would be before the actual compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. ``` ./reproduce_ci_env.py create rhel8 ./reproduce_ci_env.py run rhel8 ``` To compile VTK-m from the the interactive shell you would do the following: ``` > src]# cd build/ > build]# cmake --build . ```
This commit is contained in:
parent
450d76d914
commit
24a264fce8
@ -105,8 +105,8 @@ stages:
|
||||
- sccache --show-stats
|
||||
- "cmake --version"
|
||||
- "cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake"
|
||||
script:
|
||||
- "ctest -VV -S .gitlab/ci/ctest_configure.cmake"
|
||||
script:
|
||||
- "ctest -VV -S .gitlab/ci/ctest_build.cmake"
|
||||
- sccache --show-stats
|
||||
artifacts:
|
||||
|
@ -63,8 +63,15 @@ foreach(option IN LISTS options)
|
||||
endforeach()
|
||||
|
||||
set(CTEST_USE_LAUNCHERS "ON" CACHE STRING "")
|
||||
set(CMAKE_C_COMPILER_LAUNCHER "sccache" CACHE STRING "")
|
||||
set(CMAKE_CXX_COMPILER_LAUNCHER "sccache" CACHE STRING "")
|
||||
if(VTKm_ENABLE_CUDA)
|
||||
set(CMAKE_CUDA_COMPILER_LAUNCHER "sccache" CACHE STRING "")
|
||||
|
||||
# We need to store the absolute path so that
|
||||
# the launcher still work even when sccache isn't
|
||||
# on our path.
|
||||
find_program(SCCACHE_COMMAND NAMES sccache)
|
||||
if(SCCACHE_COMMAND)
|
||||
set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
|
||||
set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
|
||||
if(VTKm_ENABLE_CUDA)
|
||||
set(CMAKE_CUDA_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
|
||||
endif()
|
||||
endif()
|
||||
|
@ -24,7 +24,10 @@ message(STATUS "CTEST_BUILD_FLAGS: ${CTEST_BUILD_FLAGS}")
|
||||
ctest_build(APPEND
|
||||
NUMBER_WARNINGS num_warnings
|
||||
RETURN_VALUE build_result)
|
||||
ctest_submit(PARTS Build)
|
||||
|
||||
if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
|
||||
ctest_submit(PARTS Build)
|
||||
endif()
|
||||
|
||||
if (build_result)
|
||||
message(FATAL_ERROR
|
||||
|
@ -27,16 +27,22 @@ ctest_start(Experimental TRACK "${CTEST_TRACK}")
|
||||
find_package(Git)
|
||||
set(CTEST_UPDATE_VERSION_ONLY ON)
|
||||
set(CTEST_UPDATE_COMMAND "${GIT_EXECUTABLE}")
|
||||
ctest_update()
|
||||
|
||||
# Don't do updates when running via reproduce_ci_env.py
|
||||
if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
|
||||
ctest_update()
|
||||
endif()
|
||||
|
||||
# Configure the project.
|
||||
ctest_configure(APPEND
|
||||
OPTIONS "${cmake_args}"
|
||||
RETURN_VALUE configure_result)
|
||||
|
||||
# We can now submit because we've configured. This is a cmb-superbuild-ism.
|
||||
ctest_submit(PARTS Update)
|
||||
ctest_submit(PARTS Configure)
|
||||
# We can now submit because we've configured.
|
||||
if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
|
||||
ctest_submit(PARTS Update)
|
||||
ctest_submit(PARTS Configure)
|
||||
endif()
|
||||
|
||||
if (configure_result)
|
||||
message(FATAL_ERROR
|
||||
|
@ -33,7 +33,10 @@ ctest_test(APPEND
|
||||
EXCLUDE "${test_exclusions}"
|
||||
REPEAT "UNTIL_PASS:3"
|
||||
)
|
||||
ctest_submit(PARTS Test)
|
||||
|
||||
if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
|
||||
ctest_submit(PARTS Test)
|
||||
endif()
|
||||
|
||||
if (test_result)
|
||||
message(FATAL_ERROR
|
||||
|
1
Utilities/CI/.gitignore
vendored
Normal file
1
Utilities/CI/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
env/
|
37
Utilities/CI/DeveloperSetup.md
Normal file
37
Utilities/CI/DeveloperSetup.md
Normal file
@ -0,0 +1,37 @@
|
||||
#How to setup machine to use CI scripts#
|
||||
|
||||
#OSX and Unix#
|
||||
|
||||
|
||||
# Requirements #
|
||||
|
||||
- Docker
|
||||
- Python3
|
||||
-- PyYAML
|
||||
|
||||
The CI scripts require python3 and the PyYAML package.
|
||||
|
||||
Generally the best way to setup this environment is to create a python
|
||||
virtual env so you don't pollute your system. This means getting pip
|
||||
the python package manager, and virtual env which allow for isolation
|
||||
of a projects python dependencies.
|
||||
|
||||
```
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
```
|
||||
|
||||
Next we need to create a new virtual env of python. I personally
|
||||
like to setup this in `vtkm/Utilities/CI/env`.
|
||||
|
||||
```
|
||||
mkdir env
|
||||
virtualenv env
|
||||
```
|
||||
|
||||
Now all we have to do is setup the requirements:
|
||||
|
||||
```
|
||||
./env/bin/pip install -r requirements.txt
|
||||
```
|
||||
|
307
Utilities/CI/reproduce_ci_env.py
Executable file
307
Utilities/CI/reproduce_ci_env.py
Executable file
@ -0,0 +1,307 @@
|
||||
#!/bin/env python3
|
||||
|
||||
#=============================================================================
|
||||
#
|
||||
# Copyright (c) Kitware, Inc.
|
||||
# All rights reserved.
|
||||
# See LICENSE.txt for details.
|
||||
#
|
||||
# This software is distributed WITHOUT ANY WARRANTY; without even
|
||||
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
|
||||
# PURPOSE. See the above copyright notice for more information.
|
||||
#
|
||||
#===============
|
||||
|
||||
import enum
|
||||
import os
|
||||
import tempfile
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import platform
|
||||
import re
|
||||
import yaml
|
||||
|
||||
def get_root_dir():
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
#find the where .gitlab-ci.yml is located
|
||||
try:
|
||||
src_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=dir_path)
|
||||
src_root = str(src_root, 'utf-8')
|
||||
src_root = src_root.rstrip('\n')
|
||||
# Corrections in case the filename is a funny Cygwin path
|
||||
src_root = re.sub(r'^/cygdrive/([a-z])/', r'\1:/', src_root)
|
||||
return src_root
|
||||
except subprocess.CalledProcessError:
|
||||
return None
|
||||
|
||||
def extract_stage_job_from_cmdline(*args):
|
||||
if len(args) == 1:
|
||||
stage_and_job = str(args[0]).split(':')
|
||||
if len(stage_and_job) == 1:
|
||||
stage_and_job = ['build', stage_and_job[0]]
|
||||
return stage_and_job
|
||||
return args
|
||||
|
||||
def load_ci_file(ci_file_path):
|
||||
ci_state = {}
|
||||
if ci_file_path:
|
||||
root_dir = os.path.dirname(ci_file_path)
|
||||
ci_state = yaml.safe_load(open(ci_file_path))
|
||||
if 'include' in ci_state:
|
||||
for inc in ci_state['include']:
|
||||
if 'local' in inc:
|
||||
#the local paths can start with '/'
|
||||
include_path = inc['local'].lstrip('/')
|
||||
include_path = os.path.join(root_dir, include_path)
|
||||
ci_state.update(yaml.safe_load(open(include_path)))
|
||||
return ci_state
|
||||
|
||||
def ci_stages_and_jobs(ci_state):
|
||||
stages = ci_state['stages']
|
||||
jobs = dict((s,[]) for s in stages)
|
||||
for key in ci_state:
|
||||
maybe_stage = key.split(':')
|
||||
if maybe_stage[0] in stages:
|
||||
jobs[maybe_stage[0]].append(maybe_stage[1])
|
||||
return jobs
|
||||
|
||||
def subset_yml(ci_state, stage, name):
|
||||
#given a stage and name generate a new yaml
|
||||
#file that only contains information for stage and name.
|
||||
#Does basic extend merging so that recreating the env is easier
|
||||
runner_yml = {}
|
||||
yml_name = stage+":"+name
|
||||
runner_yml[yml_name] = ci_state[yml_name]
|
||||
entry = runner_yml[yml_name]
|
||||
|
||||
#Flatten 'extends' entries, only presume the first level of inheritance is
|
||||
#important
|
||||
if 'extends' in entry:
|
||||
to_merge = []
|
||||
|
||||
if not isinstance(entry['extends'], list):
|
||||
entry['extends'] = [ entry['extends'] ]
|
||||
|
||||
for e in entry['extends']:
|
||||
entry.update(ci_state[e])
|
||||
del entry['extends']
|
||||
return runner_yml
|
||||
|
||||
class CallMode(enum.Enum):
|
||||
call = 1
|
||||
output = 2
|
||||
|
||||
|
||||
def subprocess_call_docker(cmd, cwd, mode=CallMode.call):
|
||||
system = platform.system()
|
||||
if (system == 'Windows') or (system == 'Darwin'):
|
||||
# Windows and MacOS run Docker in a VM, so they don't need sudo
|
||||
full_cmd = ['docker'] + cmd
|
||||
else:
|
||||
# Unix needs to run docker with root privileges
|
||||
full_cmd = ['sudo', 'docker'] + cmd
|
||||
print(" ".join(full_cmd), flush=True)
|
||||
|
||||
if mode is CallMode.call:
|
||||
return subprocess.check_call(full_cmd, cwd=cwd)
|
||||
if mode is CallMode.output:
|
||||
return subprocess.check_output(full_cmd, cwd=cwd)
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# User Command: 'list'
|
||||
#
|
||||
###############################################################################
|
||||
def list_jobs(ci_file_path, *args):
|
||||
ci_state = load_ci_file(ci_file_path)
|
||||
jobs = ci_stages_and_jobs(ci_state)
|
||||
for key,values in jobs.items():
|
||||
print('Jobs for Stage:', key)
|
||||
for v in values:
|
||||
print('\t',v)
|
||||
print('')
|
||||
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# User Command: 'build' | 'setup'
|
||||
#
|
||||
###############################################################################
|
||||
def create_container(ci_file_path, *args):
|
||||
ci_state = load_ci_file(ci_file_path)
|
||||
ci_jobs = ci_stages_and_jobs(ci_state)
|
||||
stage,name = extract_stage_job_from_cmdline(*args)
|
||||
|
||||
if not stage in ci_jobs:
|
||||
print('Unable to find stage: ', stage)
|
||||
print('Valid stages are:', list(ci_jobs.keys()))
|
||||
exit(1)
|
||||
|
||||
if not name in ci_jobs[stage]:
|
||||
print('Unable to find job: ', name)
|
||||
print('Valid jobs are:', ci_jobs[stage])
|
||||
exit(1)
|
||||
|
||||
#we now have the relevant subset of the yml
|
||||
#fully expanded into a single definition
|
||||
subset = subset_yml(ci_state, stage, name)
|
||||
|
||||
runner_name = stage+":"+name
|
||||
runner = subset[runner_name]
|
||||
src_dir = get_root_dir()
|
||||
gitlab_env = [ k + '="' + v + '"' for k,v in runner['variables'].items()]
|
||||
|
||||
# propagate any https/http proxy info
|
||||
if os.getenv('http_proxy'):
|
||||
gitlab_env = [ 'http_proxy=' + os.getenv('http_proxy') ] + gitlab_env
|
||||
if os.getenv('https_proxy'):
|
||||
gitlab_env = [ 'https_proxy=' + os.getenv('https_proxy') ] + gitlab_env
|
||||
|
||||
# The script and before_script could be anywhere!
|
||||
script_search_locations = [ci_state, subset, runner]
|
||||
for loc in script_search_locations:
|
||||
if 'before_script' in loc:
|
||||
before_script = loc['before_script']
|
||||
if 'script' in loc:
|
||||
script = loc['script']
|
||||
|
||||
docker_template = string.Template('''
|
||||
FROM $image
|
||||
ENV GITLAB_CI=1 \
|
||||
GITLAB_CI_EMULATION=1 \
|
||||
CI_PROJECT_DIR=. \
|
||||
CI_JOB_NAME=$job_name
|
||||
#Copy all of this project to the src directory
|
||||
COPY . /src
|
||||
ENV $gitlab_env
|
||||
WORKDIR /src
|
||||
#Let git fix issues from copying across OS (such as windows EOL)
|
||||
#Note that this will remove any changes not committed.
|
||||
RUN echo "$before_script || true" >> /setup-gitlab-env.sh && \
|
||||
echo "$script || true" >> /run-gitlab-stage.sh && \
|
||||
git reset --hard && \
|
||||
bash /setup-gitlab-env.sh
|
||||
''')
|
||||
|
||||
docker_content = docker_template.substitute(image=runner['image'],
|
||||
job_name='local-build'+runner_name,
|
||||
src_dir=src_dir,
|
||||
gitlab_env= " ".join(gitlab_env),
|
||||
before_script=" && ".join(before_script),
|
||||
script=" && ".join(script))
|
||||
|
||||
# Write out the file
|
||||
docker_file = tempfile.NamedTemporaryFile(delete=False)
|
||||
docker_file.write(bytes(docker_content, 'utf-8'))
|
||||
docker_file.close()
|
||||
|
||||
# now we need to run docker and build this image with a name equal to the
|
||||
# ci name, and the docker context to be the current git repo root dir so
|
||||
# we can copy the current project src automagically
|
||||
try:
|
||||
subprocess_call_docker(['build', '-f', docker_file.name, '-t', runner_name, src_dir],
|
||||
cwd=src_dir)
|
||||
except subprocess.CalledProcessError:
|
||||
print('Unable to build the docker image for: ', runner_name)
|
||||
exit(1)
|
||||
finally:
|
||||
# remove the temp file
|
||||
os.remove(docker_file.name)
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# User Command: 'help'
|
||||
#
|
||||
###############################################################################
|
||||
def run_container(ci_file_path, *args):
|
||||
# Exec/Run ( https://docs.docker.com/engine/reference/commandline/exec/#run-docker-exec-on-a-running-container )
|
||||
src_dir = get_root_dir()
|
||||
stage,name = extract_stage_job_from_cmdline(*args)
|
||||
image_name = stage+':'+name
|
||||
|
||||
try:
|
||||
cmd = ['run', '-itd', image_name]
|
||||
container_id = subprocess_call_docker(cmd, cwd=src_dir, mode=CallMode.output)
|
||||
container_id = str(container_id, 'utf-8')
|
||||
container_id= container_id.rstrip('\n')
|
||||
except subprocess.CalledProcessError:
|
||||
print('Unable to run the docker image for: ', image_name)
|
||||
exit(1)
|
||||
|
||||
try:
|
||||
cmd = ['exec', '-it', container_id, 'bash']
|
||||
subprocess_call_docker(cmd, cwd=src_dir)
|
||||
except subprocess.CalledProcessError:
|
||||
print('Unable to attach an iteractive shell to : ', container_id)
|
||||
pass
|
||||
|
||||
try:
|
||||
cmd = ['container', 'stop', container_id]
|
||||
subprocess_call_docker(cmd, cwd=src_dir)
|
||||
except subprocess.CalledProcessError:
|
||||
print('Unable to stop container: ', container_id)
|
||||
pass
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# User Command: 'help'
|
||||
#
|
||||
###############################################################################
|
||||
def help_usage(ci_file_path, *args):
|
||||
print('Setup gitlab-ci docker environments/state locally')
|
||||
print('Usage: reproduce_ci_env.py [command] [stage] <name>')
|
||||
print('\n')
|
||||
print('Commands:\n' + \
|
||||
'\n'+\
|
||||
' list: List all stage and job names for gitlab-ci\n'+\
|
||||
' create: build a docker container for this gitlab-ci job.\n'+\
|
||||
' Will match the <stage> to docker repo, and <name> to the tag. \n' +\
|
||||
' If no explicit <stage> is provided will default to `build` stage. \n' +\
|
||||
' run: Launch an interactive shell inside the docker image\n' +\
|
||||
' for a given stage:name with the correct environment and will automatically\n' +\
|
||||
' run the associated stage script.\n'
|
||||
' If no explicit <stage> is provided will default to `build` stage. \n')
|
||||
print('Example:\n' + \
|
||||
'\n'+\
|
||||
' reproduce_ci_env create centos7\n'+\
|
||||
' reproduce_ci_env run build:centos7\n')
|
||||
|
||||
###############################################################################
|
||||
def main(argv):
|
||||
ci_file_path = os.path.join(get_root_dir(), '.gitlab-ci.yml')
|
||||
if len(argv) == 0:
|
||||
help_usage( ci_file_path )
|
||||
exit(1)
|
||||
if len(argv) > 3:
|
||||
help_usage( ci_file_path )
|
||||
exit(1)
|
||||
|
||||
#commands we want
|
||||
# - list
|
||||
# -- list all 'jobs'
|
||||
# - create | setup
|
||||
# -- create a docker image that represents a given stage:name
|
||||
# - run | exec
|
||||
# -- run the script for the stage:name inside the correct docker image
|
||||
# and provide an interactive shell
|
||||
# -- help
|
||||
#setup arg function table
|
||||
commands = {
|
||||
'list': list_jobs,
|
||||
'create': create_container,
|
||||
'setup': create_container,
|
||||
'exec': run_container,
|
||||
'run': run_container,
|
||||
'help': help_usage
|
||||
}
|
||||
if argv[0] in commands:
|
||||
#splat the subset of the vector so they are separate call parameters
|
||||
commands[argv[0]]( ci_file_path, *argv[1:3] )
|
||||
else:
|
||||
commands['help']( ci_file_path )
|
||||
exit(1)
|
||||
exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
1
Utilities/CI/requirements.txt
Normal file
1
Utilities/CI/requirements.txt
Normal file
@ -0,0 +1 @@
|
||||
PyYAML
|
@ -71,10 +71,58 @@ Current gitlab runner tags for VTK-m are:
|
||||
run the VTK-m tests
|
||||
|
||||
# How to use docker builders locally
|
||||
## Setting up docker
|
||||
|
||||
When diagnosing issues from the docker builders it can be useful to iterate locally on a
|
||||
solution.
|
||||
|
||||
If you haven't set up docker locally we recommend following the official getting started guide:
|
||||
- https://docs.docker.com/get-started/
|
||||
|
||||
|
||||
## Setting up nvidia runtime
|
||||
|
||||
To properly test VTK-m inside docker containers when the CUDA backend is enabled you will need
|
||||
to have installed the nvidia-container-runtime ( https://github.com/NVIDIA/nvidia-container-runtime )
|
||||
and be using a recent version of docker ( we recommend docker-ce )
|
||||
|
||||
|
||||
Once nvidia-container-runtime is installed you will want the default-runtime be `nvidia` so
|
||||
that `docker run` will automatically support gpus. The easiest way to do so is to add
|
||||
the following to your `/etc/docker/daemon.json`
|
||||
|
||||
```
|
||||
{
|
||||
"default-runtime": "nvidia",
|
||||
"runtimes": {
|
||||
"nvidia": {
|
||||
"path": "/usr/bin/nvidia-container-runtime",
|
||||
"runtimeArgs": []
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
## Running docker images
|
||||
|
||||
To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the
|
||||
work automatically for you.
|
||||
|
||||
The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml.
|
||||
|
||||
To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci
|
||||
worker as a docker image and setup a container just as how gitlab-ci would be before the actual
|
||||
compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell.
|
||||
|
||||
```
|
||||
./reproduce_ci_env.py create rhel8
|
||||
./reproduce_ci_env.py run rhel8
|
||||
```
|
||||
|
||||
To compile VTK-m from the the interactive shell you would do the following:
|
||||
```
|
||||
> src]# cd build/
|
||||
> build]# cmake --build .
|
||||
```
|
||||
|
||||
# How to Add/Update Kitware Gitlab CI
|
||||
|
||||
|
21
docs/changelog/ci-script.md
Normal file
21
docs/changelog/ci-script.md
Normal file
@ -0,0 +1,21 @@
|
||||
# Provide scripts to build Gitlab-ci workers locally
|
||||
|
||||
To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the
|
||||
work automatically for you.
|
||||
|
||||
The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml.
|
||||
|
||||
To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci
|
||||
worker as a docker image and setup a container just as how gitlab-ci would be before the actual
|
||||
compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell.
|
||||
|
||||
```
|
||||
./reproduce_ci_env.py create rhel8
|
||||
./reproduce_ci_env.py run rhel8
|
||||
```
|
||||
|
||||
To compile VTK-m from the the interactive shell you would do the following:
|
||||
```
|
||||
> src]# cd build/
|
||||
> build]# cmake --build .
|
||||
```
|
Loading…
Reference in New Issue
Block a user