Merge topic 'replicate_gitlab-ci_env_script'

24a264fce Add scripts to allow developers to replicate CI environments

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Kenneth Moreland <kmorel@sandia.gov>
Merge-request: !2030

.gitlab-ci.yml

@@ -105,8 +105,8 @@ stages:
     - sccache --show-stats
     - "cmake --version"
     - "cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake"
-  script:
     - "ctest -VV -S .gitlab/ci/ctest_configure.cmake"
+  script:
     - "ctest -VV -S .gitlab/ci/ctest_build.cmake"
     - sccache --show-stats
   artifacts:

.gitlab/ci/config/initial_config.cmake

@@ -63,8 +63,15 @@ foreach(option IN LISTS options)
 endforeach()
 
 set(CTEST_USE_LAUNCHERS "ON" CACHE STRING "")
-set(CMAKE_C_COMPILER_LAUNCHER "sccache" CACHE STRING "")
-set(CMAKE_CXX_COMPILER_LAUNCHER "sccache" CACHE STRING "")
-if(VTKm_ENABLE_CUDA)
-  set(CMAKE_CUDA_COMPILER_LAUNCHER "sccache" CACHE STRING "")
+
+# We need to store the absolute path so that
+# the launcher still work even when sccache isn't
+# on our path.
+find_program(SCCACHE_COMMAND NAMES sccache)
+if(SCCACHE_COMMAND)
+  set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
+  set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
+  if(VTKm_ENABLE_CUDA)
+    set(CMAKE_CUDA_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
+  endif()
 endif()

.gitlab/ci/ctest_build.cmake

@@ -24,7 +24,10 @@ message(STATUS "CTEST_BUILD_FLAGS: ${CTEST_BUILD_FLAGS}")
 ctest_build(APPEND
   NUMBER_WARNINGS num_warnings
   RETURN_VALUE build_result)
-ctest_submit(PARTS Build)
+
+if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
+  ctest_submit(PARTS Build)
+endif()
 
 if (build_result)
   message(FATAL_ERROR

.gitlab/ci/ctest_configure.cmake

@@ -27,16 +27,22 @@ ctest_start(Experimental TRACK "${CTEST_TRACK}")
 find_package(Git)
 set(CTEST_UPDATE_VERSION_ONLY ON)
 set(CTEST_UPDATE_COMMAND "${GIT_EXECUTABLE}")
-ctest_update()
+
+# Don't do updates when running via reproduce_ci_env.py
+if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
+  ctest_update()
+endif()
 
 # Configure the project.
 ctest_configure(APPEND
   OPTIONS "${cmake_args}"
   RETURN_VALUE configure_result)
 
-# We can now submit because we've configured. This is a cmb-superbuild-ism.
-ctest_submit(PARTS Update)
-ctest_submit(PARTS Configure)
+# We can now submit because we've configured.
+if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
+  ctest_submit(PARTS Update)
+  ctest_submit(PARTS Configure)
+endif()
 
 if (configure_result)
   message(FATAL_ERROR

.gitlab/ci/ctest_test.cmake

@@ -33,7 +33,10 @@ ctest_test(APPEND
   EXCLUDE "${test_exclusions}"
   REPEAT "UNTIL_PASS:3"
   )
-ctest_submit(PARTS Test)
+
+if(NOT DEFINED ENV{GITLAB_CI_EMULATION})
+  ctest_submit(PARTS Test)
+endif()
 
 if (test_result)
   message(FATAL_ERROR

Utilities/CI/.gitignore

@@ -0,0 +1 @@
+env/

Utilities/CI/DeveloperSetup.md

@@ -0,0 +1,37 @@
+#How to setup machine to use CI scripts#
+
+#OSX and Unix#
+
+
+# Requirements #
+
+- Docker
+- Python3
+-- PyYAML
+
+The CI scripts require python3 and the PyYAML package.
+
+Generally the best way to setup this environment is to create a python
+virtual env so you don't pollute your system. This means getting pip
+the python package manager, and virtual env which allow for isolation
+of a projects python dependencies.
+
+```
+sudo easy_install pip
+sudo pip install virtualenv
+```
+
+Next we need to create a new virtual env of python. I personally
+like to setup this in `vtkm/Utilities/CI/env`.
+
+```
+mkdir env
+virtualenv env
+```
+
+Now all we have to do is setup the requirements:
+
+```
+./env/bin/pip install -r requirements.txt
+```
+

Utilities/CI/reproduce_ci_env.py

@@ -0,0 +1,307 @@
+#!/bin/env python3
+
+#=============================================================================
+#
+#  Copyright (c) Kitware, Inc.
+#  All rights reserved.
+#  See LICENSE.txt for details.
+#
+#  This software is distributed WITHOUT ANY WARRANTY; without even
+#  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#  PURPOSE.  See the above copyright notice for more information.
+#
+#===============
+
+import enum
+import os
+import tempfile
+import string
+import subprocess
+import sys
+import platform
+import re
+import yaml
+
+def get_root_dir():
+  dir_path = os.path.dirname(os.path.realpath(__file__))
+  #find the where .gitlab-ci.yml is located
+  try:
+    src_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=dir_path)
+    src_root = str(src_root, 'utf-8')
+    src_root = src_root.rstrip('\n')
+    # Corrections in case the filename is a funny Cygwin path
+    src_root = re.sub(r'^/cygdrive/([a-z])/', r'\1:/', src_root)
+    return src_root
+  except subprocess.CalledProcessError:
+    return None
+
+def extract_stage_job_from_cmdline(*args):
+  if len(args) == 1:
+    stage_and_job = str(args[0]).split(':')
+    if len(stage_and_job) == 1:
+      stage_and_job = ['build', stage_and_job[0]]
+    return stage_and_job
+  return args
+
+def load_ci_file(ci_file_path):
+  ci_state = {}
+  if ci_file_path:
+    root_dir = os.path.dirname(ci_file_path)
+    ci_state = yaml.safe_load(open(ci_file_path))
+    if 'include' in ci_state:
+      for inc in ci_state['include']:
+        if 'local' in inc:
+          #the local paths can start with '/'
+          include_path = inc['local'].lstrip('/')
+          include_path = os.path.join(root_dir, include_path)
+          ci_state.update(yaml.safe_load(open(include_path)))
+  return ci_state
+
+def ci_stages_and_jobs(ci_state):
+  stages = ci_state['stages']
+  jobs = dict((s,[]) for s in stages)
+  for key in ci_state:
+    maybe_stage = key.split(':')
+    if maybe_stage[0] in stages:
+      jobs[maybe_stage[0]].append(maybe_stage[1])
+  return jobs
+
+def subset_yml(ci_state, stage, name):
+  #given a stage and name generate a new yaml
+  #file that only contains information for stage and name.
+  #Does basic extend merging so that recreating the env is easier
+  runner_yml = {}
+  yml_name = stage+":"+name
+  runner_yml[yml_name] = ci_state[yml_name]
+  entry = runner_yml[yml_name]
+
+  #Flatten 'extends' entries, only presume the first level of inheritance is
+  #important
+  if 'extends' in entry:
+    to_merge = []
+
+    if not isinstance(entry['extends'], list):
+      entry['extends'] = [ entry['extends'] ]
+
+    for e in entry['extends']:
+      entry.update(ci_state[e])
+    del entry['extends']
+  return runner_yml
+
+class CallMode(enum.Enum):
+  call = 1
+  output = 2
+
+
+def subprocess_call_docker(cmd, cwd, mode=CallMode.call):
+  system = platform.system()
+  if (system == 'Windows') or (system == 'Darwin'):
+    # Windows and MacOS run Docker in a VM, so they don't need sudo
+    full_cmd = ['docker'] + cmd
+  else:
+    # Unix needs to run docker with root privileges
+    full_cmd = ['sudo', 'docker'] + cmd
+  print(" ".join(full_cmd), flush=True)
+
+  if mode is CallMode.call:
+    return subprocess.check_call(full_cmd, cwd=cwd)
+  if mode is CallMode.output:
+    return subprocess.check_output(full_cmd, cwd=cwd)
+
+###############################################################################
+#
+#     User Command: 'list'
+#
+###############################################################################
+def list_jobs(ci_file_path, *args):
+  ci_state = load_ci_file(ci_file_path)
+  jobs = ci_stages_and_jobs(ci_state)
+  for key,values in jobs.items():
+    print('Jobs for Stage:', key)
+    for v in values:
+      print('\t',v)
+    print('')
+
+
+###############################################################################
+#
+#     User Command: 'build' | 'setup'
+#
+###############################################################################
+def create_container(ci_file_path, *args):
+  ci_state = load_ci_file(ci_file_path)
+  ci_jobs = ci_stages_and_jobs(ci_state)
+  stage,name = extract_stage_job_from_cmdline(*args)
+
+  if not stage in ci_jobs:
+    print('Unable to find stage: ', stage)
+    print('Valid stages are:', list(ci_jobs.keys()))
+    exit(1)
+
+  if not name in ci_jobs[stage]:
+    print('Unable to find job: ', name)
+    print('Valid jobs are:', ci_jobs[stage])
+    exit(1)
+
+  #we now have the relevant subset of the yml
+  #fully expanded into a single definition
+  subset = subset_yml(ci_state, stage, name)
+
+  runner_name = stage+":"+name
+  runner = subset[runner_name]
+  src_dir = get_root_dir()
+  gitlab_env = [ k + '="' + v + '"' for k,v in runner['variables'].items()]
+
+  # propagate any https/http proxy info
+  if os.getenv('http_proxy'):
+    gitlab_env = [ 'http_proxy=' + os.getenv('http_proxy') ] + gitlab_env
+  if os.getenv('https_proxy'):
+    gitlab_env = [ 'https_proxy=' + os.getenv('https_proxy') ] + gitlab_env
+
+  # The script and before_script could be anywhere!
+  script_search_locations = [ci_state, subset, runner]
+  for loc in script_search_locations:
+    if 'before_script' in loc:
+      before_script = loc['before_script']
+    if 'script' in loc:
+      script = loc['script']
+
+  docker_template = string.Template('''
+FROM $image
+ENV GITLAB_CI=1 \
+    GITLAB_CI_EMULATION=1 \
+    CI_PROJECT_DIR=. \
+    CI_JOB_NAME=$job_name
+#Copy all of this project to the src directory
+COPY . /src
+ENV $gitlab_env
+WORKDIR /src
+#Let git fix issues from copying across OS (such as windows EOL)
+#Note that this will remove any changes not committed.
+RUN echo "$before_script || true" >> /setup-gitlab-env.sh && \
+    echo "$script || true" >> /run-gitlab-stage.sh && \
+    git reset --hard && \
+    bash /setup-gitlab-env.sh
+''')
+
+  docker_content = docker_template.substitute(image=runner['image'],
+                    job_name='local-build'+runner_name,
+                    src_dir=src_dir,
+                    gitlab_env= " ".join(gitlab_env),
+                    before_script=" && ".join(before_script),
+                    script=" && ".join(script))
+
+  # Write out the file
+  docker_file = tempfile.NamedTemporaryFile(delete=False)
+  docker_file.write(bytes(docker_content, 'utf-8'))
+  docker_file.close()
+
+  # now we need to run docker and build this image with a name equal to the
+  # ci name, and the docker context to be the current git repo root dir so
+  # we can copy the current project src automagically
+  try:
+    subprocess_call_docker(['build', '-f', docker_file.name, '-t', runner_name, src_dir],
+                           cwd=src_dir)
+  except subprocess.CalledProcessError:
+    print('Unable to build the docker image for: ', runner_name)
+    exit(1)
+  finally:
+    # remove the temp file
+    os.remove(docker_file.name)
+
+###############################################################################
+#
+#     User Command: 'help'
+#
+###############################################################################
+def run_container(ci_file_path, *args):
+  # Exec/Run ( https://docs.docker.com/engine/reference/commandline/exec/#run-docker-exec-on-a-running-container )
+  src_dir = get_root_dir()
+  stage,name = extract_stage_job_from_cmdline(*args)
+  image_name = stage+':'+name
+
+  try:
+    cmd = ['run', '-itd', image_name]
+    container_id = subprocess_call_docker(cmd, cwd=src_dir, mode=CallMode.output)
+    container_id = str(container_id, 'utf-8')
+    container_id= container_id.rstrip('\n')
+  except subprocess.CalledProcessError:
+    print('Unable to run the docker image for: ', image_name)
+    exit(1)
+
+  try:
+    cmd = ['exec', '-it', container_id, 'bash']
+    subprocess_call_docker(cmd, cwd=src_dir)
+  except subprocess.CalledProcessError:
+    print('Unable to attach an iteractive shell to : ', container_id)
+  pass
+
+  try:
+    cmd = ['container', 'stop', container_id]
+    subprocess_call_docker(cmd, cwd=src_dir)
+  except subprocess.CalledProcessError:
+    print('Unable to stop container: ', container_id)
+  pass
+
+###############################################################################
+#
+#     User Command: 'help'
+#
+###############################################################################
+def help_usage(ci_file_path, *args):
+  print('Setup gitlab-ci docker environments/state locally')
+  print('Usage: reproduce_ci_env.py [command] [stage] <name>')
+  print('\n')
+  print('Commands:\n' + \
+  '\n'+\
+  '  list: List all stage and job names for gitlab-ci\n'+\
+  '  create: build a docker container for this gitlab-ci job.\n'+\
+  '        Will match the <stage> to docker repo, and <name> to the tag. \n' +\
+  '        If no explicit <stage> is provided will default to `build` stage. \n' +\
+  '  run: Launch an interactive shell inside the docker image\n' +\
+  '        for a given stage:name with the correct environment and will automatically\n' +\
+  '        run the associated stage script.\n'
+  '        If no explicit <stage> is provided will default to `build` stage. \n')
+  print('Example:\n' + \
+  '\n'+\
+  '  reproduce_ci_env create centos7\n'+\
+  '  reproduce_ci_env run build:centos7\n')
+
+###############################################################################
+def main(argv):
+  ci_file_path = os.path.join(get_root_dir(), '.gitlab-ci.yml')
+  if len(argv) == 0:
+    help_usage( ci_file_path  )
+    exit(1)
+  if len(argv) > 3:
+    help_usage( ci_file_path  )
+    exit(1)
+
+  #commands we want
+  # - list
+  # -- list all 'jobs'
+  # - create | setup
+  # -- create a docker image that represents a given stage:name
+  # - run | exec
+  # -- run the script for the stage:name inside the correct docker image
+  #    and provide an interactive shell
+  # -- help
+  #setup arg function table
+  commands = {
+    'list': list_jobs,
+    'create': create_container,
+    'setup': create_container,
+    'exec': run_container,
+    'run': run_container,
+    'help': help_usage
+    }
+  if argv[0] in commands:
+    #splat the subset of the vector so they are separate call parameters
+    commands[argv[0]]( ci_file_path, *argv[1:3] )
+  else:
+    commands['help']( ci_file_path )
+    exit(1)
+  exit(0)
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

Utilities/CI/requirements.txt

@@ -0,0 +1 @@
+PyYAML

docs/CI-README.md

@@ -71,10 +71,58 @@ Current gitlab runner tags for VTK-m are:
         run the VTK-m tests
 
 # How to use docker builders locally
-## Setting up docker
+
+When diagnosing issues from the docker builders it can be useful to iterate locally on a 
+solution.
+
+If you haven't set up docker locally we recommend following the official getting started guide:
+    - https://docs.docker.com/get-started/
+
+
 ## Setting up nvidia runtime
+
+To properly test VTK-m inside docker containers when the CUDA backend is enabled you will need
+to have installed the nvidia-container-runtime ( https://github.com/NVIDIA/nvidia-container-runtime )
+and be using a recent version of docker ( we recommend docker-ce )
+
+
+Once nvidia-container-runtime is installed you will want the default-runtime be `nvidia` so
+that `docker run` will automatically support gpus. The easiest way to do so is to add
+the following to your `/etc/docker/daemon.json`
+
+```
+{
+ "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "/usr/bin/nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    },
+}
+```
+
 ## Running docker images
 
+To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the
+work automatically for you.
+
+The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. 
+
+To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci
+worker as a docker image and setup a container just as how gitlab-ci would be before the actual
+compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. 
+
+```
+./reproduce_ci_env.py create rhel8
+./reproduce_ci_env.py run rhel8
+```
+
+To compile VTK-m from the the interactive shell you would do the following:
+```
+> src]# cd build/
+> build]# cmake --build .
+```
 
 # How to Add/Update Kitware Gitlab CI
 

docs/changelog/ci-script.md

@@ -0,0 +1,21 @@
+# Provide scripts to build Gitlab-ci workers locally
+
+To simplify reproducing docker based CI workers locally, VTK-m has python program that handles all the
+work automatically for you.
+
+The program is located in `[Utilities/CI/reproduce_ci_env.py ]` and requires python3 and pyyaml. 
+
+To use the program is really easy! The following two commands will create the `build:rhel8` gitlab-ci
+worker as a docker image and setup a container just as how gitlab-ci would be before the actual
+compilation of VTK-m. Instead of doing the compilation, instead you will be given an interactive shell. 
+
+```
+./reproduce_ci_env.py create rhel8
+./reproduce_ci_env.py run rhel8
+```
+
+To compile VTK-m from the the interactive shell you would do the following:
+```
+> src]# cd build/
+> build]# cmake --build .
+```