From b16e4fc17b4052aa6d5adee699c102f366b3dbd9 Mon Sep 17 00:00:00 2001
From: Robert Maynard <robert.maynard@kitware.com>
Date: Wed, 12 Feb 2020 16:38:57 -0500
Subject: [PATCH] Add ECP CI support to vtk-m

VTK-m now can be tested on ECP CI ( https://ecp.osti.gov/nmc )
resources. This is done via the `.gitlab-ci-ecp.yml` file
---
 .gitlab-ci-ecp.yml    | 125 ++++++++++++++++++++++++++++++++++++++++++
 .gitlab/ci/ecp/README |  18 ++++++
 2 files changed, 143 insertions(+)
 create mode 100644 .gitlab-ci-ecp.yml
 create mode 100644 .gitlab/ci/ecp/README

diff --git a/.gitlab-ci-ecp.yml b/.gitlab-ci-ecp.yml
new file mode 100644
index 000000000..f6d071c90
--- /dev/null
+++ b/.gitlab-ci-ecp.yml
@@ -0,0 +1,125 @@
+
+
+.slurm_p9_cuda: &slurm_p9_cuda
+    tags:
+        - nmc
+        - slurm
+        - nmc-xxfe1-sched-001
+        - xx-fe1
+    variables:
+        NMC_FE1_SLURM_PARAMETERS: " -N1 -p ecp-p9-4v100 --extra-node-info=*:*:* -t 1:30:00 "
+        CC: "gcc"
+        CXX: "g++"
+        CUDAHOSTCXX: "g++"
+    before_script:
+        # We need gcc-4.8.5, which is the system default compiler but not a compiler
+        # listed under the module system.
+        #
+        # That means to get this to work properly we explicitly do not request
+        # any compiler.
+        - module load cuda cmake/3.14.5
+
+
+.slurm_p9_opemp: &slurm_p9_opemp
+    tags:
+        - nmc
+        - slurm
+        - nmc-xxfe1-sched-001
+        - xx-fe1
+    variables:
+        NMC_FE1_SLURM_PARAMETERS: " -N1 -p ecp-p9-4v100 --extra-node-info=*:*:* -t 1:30:00 "
+    before_script:
+        - module load gcc/8.3.0 openmpi/3.1.4 cmake/3.14.5
+
+.cmake_build_artifacts: &cmake_build_artifacts
+    artifacts:
+        expire_in: 24 hours
+        when: always
+        paths:
+            # The artifacts of the build.
+            - vtkm-build/bin/
+            - vtkm-build/include/
+
+            # CTest files.
+            # XXX(globbing): Can be simplified with support from
+            # https://gitlab.com/gitlab-org/gitlab-runner/issues/4840
+            - vtkm-build/CTestCustom*.cmake
+            - vtkm-build/CTestTestfile.cmake
+            - vtkm-build/*/CTestTestfile.cmake
+            - vtkm-build/*/*/CTestTestfile.cmake
+            - vtkm-build/*/*/*/CTestTestfile.cmake
+            - vtkm-build/*/*/*/*/CTestTestfile.cmake
+            - vtkm-build/*/*/*/*/*/CTestTestfile.cmake
+            - vtkm-build/Testing/
+
+            # CDash files.
+            - vtkm-build/DartConfiguration.tcl
+
+.cmake_build_p9_cuda: &cmake_build_p9_cuda
+    stage: build
+    script:
+        - srun env | grep SLURM_JOB_NAME
+        - mkdir vtkm-build
+        - pushd vtkm-build
+        - cmake -DCMAKE_BUILD_TYPE=Release -DVTKm_ENABLE_CUDA=ON -S ../
+        - cmake --build . -j20
+        - popd
+
+.cmake_build_p9_openmp: &cmake_build_p9_openmp
+    stage: build
+    script:
+        - srun env | grep SLURM_JOB_NAME
+        - mkdir vtkm-build
+        - pushd vtkm-build
+        - cmake -DCMAKE_BUILD_TYPE=Release -DVTKm_ENABLE_OPENMP=ON -S ../
+        - cmake --build . -j20
+        - popd
+
+
+
+.cmake_test_p9: &cmake_test_p9
+    stage: test
+    script:
+        - echo "running the test using artifacts of the build"
+        - pushd vtkm-build
+        # We need to exclude the following tests
+        #   - CopyrightStatement
+        #   - TestInstallSetup
+        #   - SourceInInstall
+        # Which we can do by using an exclude regex
+        - ctest -E "Install|CopyrightStatement"
+        - popd
+
+stages:
+    - build
+    - test
+
+build:p9_openmp:
+    extends:
+        - .slurm_p9_opemp
+        - .cmake_build_artifacts
+        - .cmake_build_p9_openmp
+
+test:p9_openmp:
+    extends:
+        - .slurm_p9_opemp
+        - .cmake_test_p9
+    dependencies:
+        - build:p9_openmp
+    needs:
+        - build:p9_openmp
+
+build:p9_cuda:
+    extends:
+        - .slurm_p9_cuda
+        - .cmake_build_artifacts
+        - .cmake_build_p9_cuda
+
+test:p9_cuda:
+    extends:
+        - .slurm_p9_cuda
+        - .cmake_test_p9
+    dependencies:
+        - build:p9_cuda
+    needs:
+        - build:p9_cuda
diff --git a/.gitlab/ci/ecp/README b/.gitlab/ci/ecp/README
new file mode 100644
index 000000000..ffc8a27bc
--- /dev/null
+++ b/.gitlab/ci/ecp/README
@@ -0,0 +1,18 @@
+
+
+`.gitlab-ci-ecp.yml` allows for VTK-m to run CI on provided by ECP at NMC.
+
+To have this work properly you will need to make sure that the gitlab repository
+has been updated to this non-standard yaml file location
+( "Settings" -> "CI/CD" -> "General pipelines" -> "Custom CI configuration path").
+
+The ECP CI is setup to verify VTK-m mainly on Power9 hardware as that currently is
+missing from VTK-m standard CI infrastructure.
+
+Currently we verify Power9 support with `cuda` and `openmp` builders. The `cuda` builder
+is setup to use the default cuda SDK on the machine and the required `c++` compiler which
+currently is `gcc-4.8.5`. The `openmp` builder is setup to use the newest `c++` compiler provided
+on the machine so that we maximimze compiler coverage.
+
+Currently the largest outstanding issue is that these builders don't report back to the
+VTK-m CDash instance.