Add gitlab-ci infrastructure

2020-03-11 14:16:04 -04:00 · 2020-03-11 14:16:04 -04:00 · 5bc09b976e
commit 5bc09b976e
parent ef7ef5fb22
12 changed files with 745 additions and 31 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -0,0 +1,143 @@
+
+# Docker Images:
+#
+# - CentOS 7
+#   * .gitlab/ci/docker/centos7/cuda10.2/
+#     - cuda
+#     - gcc 4.8.5
+#   * .gitlab/ci/rhel8/cuda10.2/
+#     - cuda
+#     - gcc 8.2.1
+#   * .gitlab/ci/ubuntu1604/base/
+#     - gcc 4.8
+#     - clang 3.8
+#     - clang 5.0
+#     - tbb
+#     - openmpi
+#   * .gitlab/ci/ubuntu1604/cuda9.2/
+#     - cuda
+#     - gcc 5.4
+#     - tbb
+#     - openmp
+#     - openmpi
+#   * .gitlab/ci/ubuntu1804/base/
+#     - gcc 7.4
+#     - gcc 9
+#     - clang 8
+#     - tbb
+#     - openmp
+#     - mpich2
+#   * .gitlab/ci/ubuntu1804/cuda10.1/
+#     - cuda
+#     - gcc 7.4
+#     - tbb
+#     - openmp
+#     - mpich2
+
+.docker_image: &docker_image
+  variables:
+    GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci
+
+.centos7: &centos7
+  image: "kitware/vtkm:ci-centos7_cuda10.2-20200320"
+  extends:
+    - .docker_image
+
+.rhel8: &rhel8
+  image: "kitware/vtkm:ci-rhel8_cuda10.2-20200320"
+  extends:
+    - .docker_image
+
+.ubuntu1604: &ubuntu1604
+  image: "kitware/vtkm:ci-ubuntu1604-20200320"
+  extends:
+    - .docker_image
+
+.ubuntu1604_cuda: &ubuntu1604_cuda
+  image: "kitware/vtkm:ci-ubuntu1604_cuda9.2-20200320"
+  extends:
+    - .docker_image
+
+.ubuntu1804: &ubuntu1804
+  image: "kitware/vtkm:ci-ubuntu1804-20200320"
+  extends:
+    - .docker_image
+
+.ubuntu1804_cuda: &ubuntu1804_cuda
+  image: "kitware/vtkm:ci-ubuntu1804_cuda10.1-20200320"
+  extends:
+    - .docker_image
+
+
+# General Longer Term Tasks:
+# - setup asan, and ubsan as sub-pipeline
+# - setup clang tidy as sub-pipeline
+#
+# Current Tasks:
+# - Determine if we can get sccache to work with CUDA
+# - Setup a machine to replicate the issue in https://gitlab.kitware.com/vtk/vtk-m/issues/447
+#   Note: Centos7 doesn't work as they ship separate standard library
+#   headers for each version. We will have to figure out something else
+#   like using spack or building llvm/clang from source
+
+stages:
+  - build
+  - test
+
+.cmake_build_linux: &cmake_build_linux
+  stage: build
+  timeout:  2 hours
+  interruptible: true
+  before_script:
+    - .gitlab/ci/config/sccache.sh
+    - export PATH=$PWD/.gitlab:$PATH
+    - SCCACHE_IDLE_TIMEOUT=1200 sccache --start-server
+    - sccache --show-stats
+    - "cmake --version"
+    - "cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake"
+  script:
+    - "ctest -VV -S .gitlab/ci/ctest_configure.cmake"
+    - "ctest -VV -S .gitlab/ci/ctest_build.cmake"
+    - sccache --show-stats
+  artifacts:
+    expire_in: 24 hours
+    when: always
+    paths:
+      # The artifacts of the build.
+      - build/bin/
+      - build/include/
+      - build/lib/
+
+      # CTest and CMake install files.
+      # XXX(globbing): Can be simplified with support from
+      # https://gitlab.com/gitlab-org/gitlab-runner/issues/4840
+      #
+      # Note: this also captures our CIState.cmake file
+      - build/CMakeCache.txt
+      - build/*.cmake
+      - build/*/*.cmake
+      - build/*/*/*.cmake
+      - build/*/*/*/*.cmake
+      - build/*/*/*/*/*.cmake
+      - build/*/*/*/*/*/*.cmake
+      - build/Testing/
+
+      # CDash files.
+      - build/DartConfiguration.tcl
+
+.cmake_test_linux: &cmake_test_linux
+  stage: test
+  timeout:  30 minutes
+  interruptible: true
+  script:
+    #Need to use our custom ctest-latest symlink
+    #This will allow us to use 3.17+ which has support
+    #for running failed tests multiple times so failures
+    #due to system load are not reported
+    - "ctest-latest -VV -S .gitlab/ci/ctest_test.cmake"
+
+include:
+  - local: '/.gitlab/ci/centos7.yml'
+  - local: '/.gitlab/ci/rhel8.yml'
+  - local: '/.gitlab/ci/ubuntu1604.yml'
+  - local: '/.gitlab/ci/ubuntu1804.yml'
--- a/.gitlab/ci/centos7.yml
+++ b/.gitlab/ci/centos7.yml
@ -0,0 +1,49 @@
+
+# Build on centos7 with CUDA and test on rhel8 and centos7
+# gcc 4.8
+build:centos7_gcc48:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+    - large-memory
+  extends:
+    - .centos7
+    - .cmake_build_linux
+  variables:
+    CMAKE_BUILD_TYPE: RelWithDebInfo
+    CMAKE_GENERATOR: "Unix Makefiles"
+    VTKM_SETTINGS: "cuda+turing+32bit_ids"
+
+test:centos7_gcc48:
+  tags:
+    - test
+    - cuda-rt
+    - turing
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .centos7
+    - .cmake_test_linux
+  dependencies:
+    - build:centos7_gcc48
+  needs:
+    - build:centos7_gcc48
+
+test:rhel8_test_centos7:
+  tags:
+    - test
+    - cuda-rt
+    - turing
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .rhel8
+    - .cmake_test_linux
+  dependencies:
+    - build:centos7_gcc48
+  needs:
+    - build:centos7_gcc48
--- a/.gitlab/ci/config/gitlab_ci_setup.cmake
+++ b/.gitlab/ci/config/gitlab_ci_setup.cmake
@ -0,0 +1,82 @@
+##=============================================================================
+##
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##
+##=============================================================================
+
+if (NOT DEFINED "ENV{GITLAB_CI}")
+  message(FATAL_ERROR
+    "This script assumes it is being run inside of GitLab-CI")
+endif ()
+
+# Set up the source and build paths.
+set(CTEST_SOURCE_DIRECTORY "$ENV{CI_PROJECT_DIR}")
+set(CTEST_BINARY_DIRECTORY "${CTEST_SOURCE_DIRECTORY}/build")
+
+if ("$ENV{VTKM_SETTINGS}" STREQUAL "")
+  message(FATAL_ERROR
+    "The VTKM_SETTINGS environment variable is required to know what "
+    "build options should be used.")
+endif ()
+
+# Default to Release builds.
+if (NOT "$ENV{CMAKE_BUILD_TYPE}" STREQUAL "")
+  set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_BUILD_TYPE}")
+endif ()
+if (NOT CTEST_BUILD_CONFIGURATION)
+  set(CTEST_BUILD_CONFIGURATION "Release")
+endif ()
+
+# Set the build metadata.
+string(TOLOWER ${CTEST_BUILD_CONFIGURATION} build_type)
+set(CTEST_BUILD_NAME "${build_type}+$ENV{VTKM_SETTINGS}")
+
+set(site_name "$ENV{CI_JOB_NAME}")
+string(REPLACE "build" "" site_name "${site_name}")
+string(REPLACE "test" "" site_name "${site_name}")
+string(REPLACE ":" "" site_name "${site_name}")
+set(CTEST_SITE ${site_name})
+
+# Default to using Ninja.
+if (NOT "$ENV{CMAKE_GENERATOR}" STREQUAL "")
+  set(CTEST_CMAKE_GENERATOR "$ENV{CMAKE_GENERATOR}")
+endif ()
+if (NOT CTEST_CMAKE_GENERATOR)
+  set(CTEST_CMAKE_GENERATOR "Ninja")
+endif ()
+
+# Determine the track to submit to.
+set(CTEST_TRACK "Experimental")
+if ("$ENV{CI_COMMIT_REF_NAME}" STREQUAL "master")
+  set(CTEST_TRACK "Nightly")
+endif ()
+
+if (CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles")
+  include(ProcessorCount)
+  ProcessorCount(nproc)
+  set(CTEST_BUILD_FLAGS "-j${nproc}")
+endif ()
+
+#We need to do write this information out to a file in the build directory
+set(state
+"
+  set(CTEST_SOURCE_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\")
+  set(CTEST_BINARY_DIRECTORY \"${CTEST_BINARY_DIRECTORY}\")
+
+  set(CTEST_BUILD_NAME ${CTEST_BUILD_NAME})
+  set(CTEST_SITE ${CTEST_SITE})
+
+  set(CTEST_CMAKE_GENERATOR \"${CTEST_CMAKE_GENERATOR}\")
+  set(CTEST_BUILD_CONFIGURATION ${CTEST_BUILD_CONFIGURATION})
+  set(CTEST_BUILD_FLAGS \"${CTEST_BUILD_FLAGS}\")
+
+  set(CTEST_TRACK ${CTEST_TRACK})
+"
+)
+file(WRITE ${CTEST_BINARY_DIRECTORY}/CIState.cmake "${state}")
--- a/.gitlab/ci/config/initial_config.cmake
+++ b/.gitlab/ci/config/initial_config.cmake
@ -0,0 +1,61 @@
+##=============================================================================
+##
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##
+##=============================================================================
+
+string(REPLACE "+" ";" options "$ENV{VTKM_SETTINGS}")
+
+foreach(option IN LISTS options)
+
+  if(static STREQUAL option)
+    set(BUILD_SHARED_LIBS "OFF" CACHE STRING "")
+
+  elseif(shared STREQUAL option)
+    set(BUILD_SHARED_LIBS "ON" CACHE STRING "")
+
+  elseif(32bit_ids STREQUAL option)
+    set(VTKm_USE_64BIT_IDS "OFF" CACHE STRING "")
+
+  elseif(64bit_floats STREQUAL option)
+    set(VTKm_USE_DOUBLE_PRECISION "ON" CACHE STRING "")
+
+  elseif(mpi STREQUAL option)
+    set(VTKm_ENABLE_MPI "ON" CACHE STRING "")
+
+  elseif(tbb STREQUAL option)
+    set(VTKm_ENABLE_TBB "ON" CACHE STRING "")
+
+  elseif(openmp STREQUAL option)
+    set(VTKm_ENABLE_OPENMP "ON" CACHE STRING "")
+
+  elseif(cuda STREQUAL option)
+    set(VTKm_ENABLE_CUDA "ON" CACHE STRING "")
+
+  elseif(maxwell STREQUAL option)
+    set(VTKm_CUDA_Architecture "maxwell" CACHE STRING "")
+
+  elseif(pascal STREQUAL option)
+    set(VTKm_CUDA_Architecture "pascal" CACHE STRING "")
+
+  elseif(volta STREQUAL option)
+    set(VTKm_CUDA_Architecture "volta" CACHE STRING "")
+
+  elseif(turing STREQUAL option)
+    set(VTKm_CUDA_Architecture "turing" CACHE STRING "")
+  endif()
+
+endforeach()
+
+set(CTEST_USE_LAUNCHERS "ON" CACHE STRING "")
+set(CMAKE_C_COMPILER_LAUNCHER "sccache" CACHE STRING "")
+set(CMAKE_CXX_COMPILER_LAUNCHER "sccache" CACHE STRING "")
+if(VTKm_ENABLE_CUDA)
+  set(CMAKE_CUDA_COMPILER_LAUNCHER "sccache" CACHE STRING "")
+endif()
--- a/.gitlab/ci/config/sccache.sh
+++ b/.gitlab/ci/config/sccache.sh
@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+readonly version="0.2.12"
+readonly sha256sum="26fd04c1273952cc2a0f359a71c8a1857137f0ee3634058b3f4a63b69fc8eb7f"
+readonly filename="sccache-$version-x86_64-unknown-linux-musl"
+readonly tarball="$filename.tar.gz"
+
+cd .gitlab
+
+echo "$sha256sum  $tarball" > sccache.sha256sum
+curl -OL "https://github.com/mozilla/sccache/releases/download/$version/$tarball"
+sha256sum --check sccache.sha256sum
+tar xf "$tarball"
+mv "$filename/sccache" .
--- a/.gitlab/ci/ctest_build.cmake
+++ b/.gitlab/ci/ctest_build.cmake
@ -0,0 +1,32 @@
+##=============================================================================
+##
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##
+##=============================================================================
+
+cmake_minimum_required(VERSION 3.8)
+
+# Read the files from the build directory that contain
+# host information ( name, parallel level, etc )
+include("$ENV{CI_PROJECT_DIR}/build/CIState.cmake")
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+
+
+# Pick up from where the configure left off.
+ctest_start(APPEND)
+message(STATUS "CTEST_BUILD_FLAGS: ${CTEST_BUILD_FLAGS}")
+ctest_build(APPEND
+  NUMBER_WARNINGS num_warnings
+  RETURN_VALUE build_result)
+ctest_submit(PARTS Build)
+
+if (build_result)
+  message(FATAL_ERROR
+    "Failed to build")
+endif ()
--- a/.gitlab/ci/ctest_configure.cmake
+++ b/.gitlab/ci/ctest_configure.cmake
@ -0,0 +1,44 @@
+##=============================================================================
+##
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##
+##=============================================================================
+
+cmake_minimum_required(VERSION 3.8)
+
+# Read the files from the build directory that contain
+# host information ( name, parallel level, etc )
+include("$ENV{CI_PROJECT_DIR}/build/CIState.cmake")
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+
+set(cmake_args
+  -C "${CMAKE_CURRENT_LIST_DIR}/config/initial_config.cmake")
+
+# Create an entry in CDash.
+ctest_start(Experimental TRACK "${CTEST_TRACK}")
+
+# Gather update information.
+find_package(Git)
+set(CTEST_UPDATE_VERSION_ONLY ON)
+set(CTEST_UPDATE_COMMAND "${GIT_EXECUTABLE}")
+ctest_update()
+
+# Configure the project.
+ctest_configure(APPEND
+  OPTIONS "${cmake_args}"
+  RETURN_VALUE configure_result)
+
+# We can now submit because we've configured. This is a cmb-superbuild-ism.
+ctest_submit(PARTS Update)
+ctest_submit(PARTS Configure)
+
+if (configure_result)
+  message(FATAL_ERROR
+    "Failed to configure")
+endif ()
--- a/.gitlab/ci/ctest_test.cmake
+++ b/.gitlab/ci/ctest_test.cmake
@ -0,0 +1,41 @@
+##=============================================================================
+##
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##
+##=============================================================================
+
+# Read the files from the build directory that contain
+# host information ( name, parallel level, etc )
+include("$ENV{CI_PROJECT_DIR}/build/CIState.cmake")
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+
+# Pick up from where the configure left off.
+ctest_start(APPEND)
+
+set(test_exclusions
+  # placeholder for tests to exclude
+)
+
+string(REPLACE ";" "|" test_exclusions "${test_exclusions}")
+if (test_exclusions)
+  set(test_exclusions "(${test_exclusions})")
+endif ()
+
+ctest_test(APPEND
+  PARALLEL_LEVEL "10"
+  RETURN_VALUE test_result
+  EXCLUDE "${test_exclusions}"
+  REPEAT "UNTIL_PASS:3"
+  )
+ctest_submit(PARTS Test)
+
+if (test_result)
+  message(FATAL_ERROR
+    "Failed to test")
+endif ()
--- a/.gitlab/ci/rhel8.yml
+++ b/.gitlab/ci/rhel8.yml
@ -0,0 +1,29 @@
+
+# Build on rhel8 with serial and test on rhel8
+# Uses gcc 8.2.1
+build:rhel8:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .rhel8
+    - .cmake_build_linux
+  variables:
+    CMAKE_GENERATOR: "Unix Makefiles"
+    VTKM_SETTINGS: "serial+shared+64bit_floats+32bit_ids"
+
+test:rhel8:
+  tags:
+    - test
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .rhel8
+    - .cmake_test_linux
+  dependencies:
+    - build:rhel8
+  needs:
+    - build:rhel8
--- a/.gitlab/ci/ubuntu1604.yml
+++ b/.gitlab/ci/ubuntu1604.yml
@ -0,0 +1,93 @@
+
+# Build on ubuntu1604 with CUDA 9.2 and test on ubuntu1604 and ubuntu1804
+# Uses gcc 5, and build for pascal as CUDA 9.2 doesn't support turing
+build:ubuntu1604_gcc5:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+    - large-memory
+  extends:
+    - .ubuntu1604_cuda
+    - .cmake_build_linux
+  variables:
+    CMAKE_BUILD_TYPE: RelWithDebInfo
+    VTKM_SETTINGS: "cuda+pascal"
+
+# Temporarily disabled as we don't have a pascal hw gitlab-runner
+# test:ubuntu1604_gcc5:
+#   tags:
+#     - test
+#     - cuda-rt
+#     - pascal
+#     - vtkm
+#     - docker
+#     - linux
+#   extends:
+#     - .ubuntu1604_cuda
+#     - .cmake_test_linux
+#   dependencies:
+#     - build:ubuntu1604_gcc5
+#   needs:
+#     - build:ubuntu1604_gcc5
+# test:ubuntu1804_test_ubuntu1604_gcc5:
+#   tags:
+#     - test
+#     - cuda-rt
+#     - pascal
+#     - vtkm
+#     - docker
+#     - linux
+#   extends:
+#     - .ubuntu1804_cuda
+#     - .cmake_test_linux
+#   dependencies:
+#     - build:ubuntu1604_gcc5
+#   needs:
+#     - build:ubuntu1604_gcc5
+
+# Build on ubuntu1604 with mpi + tbb and test on ubuntu1604
+# Uses gcc 4.8
+# Uses OpenMPI
+build:ubuntu1604_gcc48:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+  extends:
+    #Requires the cuda docker image as it uses a sufficiently new
+    #enough CMake version that supports mpi
+    - .ubuntu1604_cuda
+    - .cmake_build_linux
+  variables:
+    CMAKE_BUILD_TYPE: Release
+    #custom openmpi install location
+    CMAKE_PREFIX_PATH: "/opt/openmpi/"
+    VTKM_SETTINGS: "tbb+mpi"
+
+test:ubuntu1604_gcc48:
+  tags:
+    - test
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1604
+    - .cmake_test_linux
+  variables:
+    #env flags to allow openmpi to run as root user
+    OMPI_ALLOW_RUN_AS_ROOT: 1
+    OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+
+  dependencies:
+    - build:ubuntu1604_gcc48
+  needs:
+    - build:ubuntu1604_gcc48
+
+# Todo:
+# Build on ubuntu1604 with TBB and test on ubuntu1804
+# Uses clang 3.8
+# Build on ubuntu1604 with TBB and test on ubuntu1604 and ubuntu1804
+# Uses clang 5.0, Debug??, Static/shared?
--- a/.gitlab/ci/ubuntu1804.yml
+++ b/.gitlab/ci/ubuntu1804.yml
@ -0,0 +1,97 @@
+
+# Build on ubuntu1804 with TBB and OpenMP and test on ubuntu1804
+# Uses gcc 9
+# Uses MPICH2
+build:ubuntu1804_gcc9:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1804
+    - .cmake_build_linux
+  variables:
+    CC: "gcc-9"
+    CXX: "g++-9"
+    CMAKE_BUILD_TYPE: Debug
+    VTKM_SETTINGS: "tbb+openmp+mpi"
+
+test:ubuntu1804_gcc9:
+  tags:
+    - test
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1804
+    - .cmake_test_linux
+  dependencies:
+    - build:ubuntu1804_gcc9
+  needs:
+    - build:ubuntu1804_gcc9
+
+# Build on ubuntu1804 with CUDA + MPI and test on ubuntu1804
+# Uses gcc 7.4
+# Uses MPICH2
+build:ubuntu1804_gcc7:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+    - large-memory
+  extends:
+    - .ubuntu1804_cuda
+    - .cmake_build_linux
+  variables:
+    CC: "gcc-7"
+    CXX: "g++-7"
+    VTKM_SETTINGS: "cuda+turing+mpi+64bit_floats"
+
+test:ubuntu1804_gcc7:
+  tags:
+    - test
+    - cuda-rt
+    - turing
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1804_cuda
+    - .cmake_test_linux
+  dependencies:
+    - build:ubuntu1804_gcc7
+  needs:
+    - build:ubuntu1804_gcc7
+
+# Build on ubuntu1804 with TBB and test on ubuntu1804
+# Uses clang 8
+build:ubuntu1804_clang8:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1804
+    - .cmake_build_linux
+  variables:
+    CC: "clang-8"
+    CXX: "clang++-8"
+    CMAKE_BUILD_TYPE: Debug
+    VTKM_SETTINGS: "tbb+shared"
+
+test:ubuntu1804_clang8:
+  tags:
+    - test
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu1804
+    - .cmake_test_linux
+  dependencies:
+    - build:ubuntu1804_clang8
+  needs:
+    - build:ubuntu1804_clang8
--- a/.gitlab/ci/README.md
+++ b/.gitlab/ci/README.md
@ -5,7 +5,6 @@ Gitlab CI
 # High level view
 1. Kitware Gitlab CI
    - Why pipelines
-    - Existing build and test pipelines
    - Gitlab runner tags

 2. How to use docker builders locally
@ -16,7 +15,6 @@ Gitlab CI
 3. How to Add/Update Kitware Gitlab CI
    - How to add a new builder
    - How to add a new tester
-    - Docker image structure
    - How to update an existing docker image

 4. ECP OSTI CI
@ -37,22 +35,15 @@ GitLab CI/CD is configured by a file called `.gitlab-ci.yml` located at the root
 The scripts set in this file are executed by the [GitLab Runners](https://docs.gitlab.com/runner/) associated with VTK-m.

 ## Why pipelines
-How this separates our the build and test components of the pipeline

-Pipelines are the top-level component of continuous integration, delivery, and deployment.
+Pipelines are the top-level component of continuous integration. For VTK-m the pipeline contains build and test stages, with the possibilty of adding subsequent stages such as coverage, or memory checking.

-Pipelines comprise:
+Decomposing the build and test into separate components comes with some significant benifits for VTK-m developers.
+The most impactful change is that we now have the ability to compile VTK-m on dedicated 'compilation' machines and
+test on machines with less memory or an older CPU improving turnaround time. Additionally since we are heavily
+leveraging docker, VTK-m build stages can be better load balanced across the set of builders as we don't have
+a tight coupling between a machine and build configuration.

-Jobs that define what to run. For example, code compilation or test runs.
-Stages that define when and how to run. For example, that tests run only after code compilation.
-Multiple jobs in the same stage are executed by Runners in parallel, if there are enough concurrent Runners.
-
-If all the jobs in a stage:
-
-Succeed, the pipeline moves on to the next stage.
-Fail, the next stage is not (usually) executed and the pipeline ends early.
-
-## Existing build and test pipelines
 ## Gitlab runner tags

 Current gitlab runner tags for VTK-m are:
@ -87,28 +78,35 @@ Current gitlab runner tags for VTK-m are:

 # How to Add/Update Kitware Gitlab CI

-## How to add a new builder
+Adding new build or test stages is necessary when a given combination of compiler, platform,
+and VTK-m options isn't already captured by existing builders. Each definition is composed via 3 components; tags, variables, and extends.

-Adding builders is necessary when a given combination of compiler, platform,
-and VTK-m options isn't already captured by existing builders.
+Tags are used to by gitlab-ci to match a given build to a set of possible execution locations.
+Therefore we encode information such as we require docker or the linux kernel into tags.
+The full set of VTK-m tags each meaning are found under the `runner tags` section of the document.
+
+Extends is used to compose the execution enviornment of the builder. Basically this means
+setting up the correct build/test enviornment and specifying the CMake scripts that need
+to be executed. So a linux docker based builder would extend the docker image they want,
+plus `.cmake_build_linux`. A MacOS builder would extend `.cmake_build_macos`.
+
+Variables control stage specific information such as runtime enviornment variables,
+or VTK-m CMake options.
+
+## How to add a new builder

 Each builder definition is placed inside the respective OS `yml` file located in
 `.gitlab/ci/`. Therefore if you are adding a builder that will run on Ubuntu 20.04 it
 would go into `.gitlab/ci/ubuntu2004.yml`.

-As each builder tests a given set of flags, we need to encode them in the yml definition.
-This information is encoded via 3 ways; tags, variables, and extends.
+Variables are used to control the following components:

-Tags are used to by gitlab-ci to match a given build to a set of possible execution locations.
-Therefore we encode information such as we require docker or the linux kernel into tags.
-The full set of VTK-m tags each meaning are found under the `Builder tags` section of the document.
+    - Compiler
+    - VTK-m CMake Options
+    - Static / Shared
+    - Release / Debug / MinSizeRel

-Extends is used to compose the actual execution component of the builder with any information.
-So a linux docker based builder would extend the docker image they want, plus `.cmake_build_linux`. A MacOS builder would extend `.cmake_build_macos`.
-
-
-Variables are used to
-The defitinon of the builder would look roughly like
+An example defitinon of a builder would look like:
 ```yml
 build:ubuntu2004_$<compiler>:
  tags:
@ -152,7 +150,37 @@ files.
 ```

 ## How to add a new tester
-## Docker image structure
+
+Each test definition is placed inside the respective OS `yml` file located in
+`.gitlab/ci/`. Therefore if you are adding a builder that will run on Ubuntu 20.04 it
+would go into `.gitlab/ci/ubuntu2004.yml`.
+
+The primary difference between tests and build definitions are that tests have
+the dependencies and needs sections. These are required as by default
+gitlab-ci will not run any test stage before ALL the build stages have
+completed.
+
+Variables for testers are currently only used for the following things:
+    - Allowing OpenMPI to run as root
+
+An example defitinon of a tester would look like:
+```yml
+test:ubuntu2004_$<compiler>:
+  tags:
+    - test
+    - cuda-rt
+    - turing
+    - vtkm
+    - docker
+    - linux
+  extends:
+    - .ubuntu2004_cuda
+    - .cmake_test_linux
+  dependencies:
+    - build:ubuntu2004_$<compiler>
+  needs:
+    - build:ubuntu2004_$<compiler>
+```

 ## How to update an existing docker image

@ -200,4 +228,3 @@ on the machine so that we maximimze compiler coverage.

 ## Issues
 Currently these builders don't report back to the VTK-m CDash instance.
-