CI: Enable Ascent builds in MR

.gitlab-ci.yml

@@ -138,6 +138,12 @@
       when: on_success
     - when: never
 
+.run_spock_ci: &run_spock_ci
+  rules:
+    - if: '$CI_PROJECT_PATH == "ci/csc331_crusher/dev/vtk-m"'
+      when: on_success
+    - when: never
+
 # General Longer Term Tasks:
 # - Setup clang tidy as sub-pipeline
 # - Setup a machine to replicate the issue in https://gitlab.kitware.com/vtk/vtk-m/-/issues/447
@@ -250,6 +256,7 @@ stages:
 
 include:
   - local: '/.gitlab/ci/ascent.yml'
+  - local: '/.gitlab/ci/spock.yml'
   - local: '/.gitlab/ci/centos7.yml'
   - local: '/.gitlab/ci/centos8.yml'
   - local: '/.gitlab/ci/doxygen.yml'

.gitlab/ci/config/initial_config.cmake

@@ -115,10 +115,7 @@ foreach(option IN LISTS options)
       message(FATAL_ERROR "VTK-m requires cmake > 3.20 to enable HIP support")
     endif()
 
-    set(CMAKE_HIP_COMPILER  "/opt/rocm/llvm/bin/clang++" CACHE FILEPATH "")
-    set(Kokkos_CXX_COMPILER "/opt/rocm/llvm/bin/clang++" CACHE FILEPATH "")
     set(VTKm_ENABLE_KOKKOS_HIP ON CACHE STRING "")
-    set(CMAKE_HIP_ARCHITECTURES "gfx900" CACHE STRING "")
 
     # -O1 and -O2 results in ridiculous build times in ROCm < 5.3
     set(CMAKE_HIP_FLAGS "-O0 " CACHE STRING "")

.gitlab/ci/spock.yml

@@ -0,0 +1,104 @@
+# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Spock.
+.spock_gcc_hip:
+  variables:
+    CCACHE_BASEDIR:          "/gpfs/alpine/world-shared/csc331/"
+    CCACHE_DIR:              "/gpfs/alpine/world-shared/csc331/vtk-m/ci/ccache"
+    CUSTOM_CI_BUILDS_DIR:    "/gpfs/alpine/world-shared/csc331/vtk-m/ci/runtime"
+
+    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
+    CCACHE_IGNOREOPTIONS:    "-isystem=*"
+    CCACHE_NOHASHDIR:        "true"
+    CCACHE_INSTALL_DIR:      "$CI_PROJECT_DIR/build"
+
+    CMAKE_BUILD_TYPE:        "RelWithDebInfo"
+    CMAKE_GENERATOR:         "Ninja"
+
+    CMAKE_HIP_COMPILER:      "/opt/rocm-default/llvm/bin/clang++"
+    Kokkos_CXX_COMPILER:     "/opt/rocm-default/llvm/bin/clang++"
+    CMAKE_HIP_ARCHITECTURES: "gfx908"
+
+    CC: gcc
+    CXX: g++
+    # DefApps/default;craype;rocm;gcc should be loaded first
+    JOB_MODULES: >-
+      DefApps/default
+      craype-accel-amd-gfx90a
+      rocm/5
+      gcc/10
+      cmake/3.22
+      git
+      git-lfs
+      kokkos/3.5.00
+      lsf-tools
+      ninja
+      spectrum-mpi
+      zstd
+    VTKM_SETTINGS: kokkos+hip+spock+ccache+no_virtual+no_rendering
+  interruptible: true
+
+.setup_env_ecpci: &setup_env_ecpci |
+  module purge
+  module load ${JOB_MODULES}
+  module list
+  export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"
+
+build:spock_gcc_hip:
+  stage: build
+  tags: [spock, shell]
+  extends:
+    - .spock_gcc_hip
+    - .run_spock_ci
+  before_script:
+    - *setup_env_ecpci
+    - mkdir -p "$CCACHE_INSTALL_DIR"
+    - cmake --version
+    - cmake -VV -P .gitlab/ci/config/ccache.cmake
+    - ccache -z
+    - ccache -s
+
+    - git remote add lfs https://gitlab.kitware.com/vtk/vtk-m.git
+    - git fetch lfs
+    - git-lfs install
+    - git-lfs pull lfs
+  script:
+    - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake
+    - ctest -VV -S .gitlab/ci/ctest_configure.cmake
+  artifacts:
+    expire_in: 24 hours
+    when: always
+    paths:
+      - build/
+      - .gitlab/ccache/ccache
+
+test:spock_gcc_hip:
+  stage: test
+  tags: [spock, slurm]
+  extends:
+    - .spock_gcc_hip
+    - .run_spock_ci
+  needs:
+    - build:spock_gcc_hip
+  dependencies:
+    - build:spock_gcc_hip
+  variables:
+    SCHEDULER_PARAMETERS: "-ACSC331 -pecp -t120 --nice=0 -c32 --gpus=4 -N 1"
+    # We need this to skip ctest_submit from being run inside a jsrun job
+    GITLAB_CI_EMULATION: 1
+    # Tests errors to address due to different env in Spock
+    # Refer to issue: https://gitlab.kitware.com/vtk/vtk-m/-/issues/743
+    CTEST_EXCLUSIONS: >-
+      UnitTestMIRFilter
+      UnitTestDistributedBranchDecompositionFilter
+      UnitTestWorkletParticleAdvection
+      UnitTestLagrangianFilter
+  before_script:
+    - *setup_env_ecpci
+  script:
+    - CTEST_MAX_PARALLELISM=32 ctest -VV -S .gitlab/ci/ctest_build.cmake
+    - CTEST_MAX_PARALLELISM=4  ctest -VV -S .gitlab/ci/ctest_test.cmake
+  after_script:
+    - *setup_env_ecpci
+    - ccache -s
+    - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake
+    - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake
+  timeout: 150 minutes

.gitlab/ci/ubuntu2004.yml

@@ -71,17 +71,20 @@ build:ubuntu2004_hip_kokkos:
     - .cmake_build_linux
     - .run_automatically
   variables:
-    CMAKE_BUILD_TYPE: RelWithDebInfo
-    VTKM_SETTINGS: "benchmarks+kokkos+hip+no_virtual+no_rendering+ccache"
-    CMAKE_PREFIX_PATH: "/opt/rocm/lib/cmake"
+    CMAKE_BUILD_TYPE:        "RelWithDebInfo"
+    VTKM_SETTINGS:           "benchmarks+kokkos+hip+no_virtual+no_rendering+ccache"
 
-    CCACHE_BASEDIR: "$CI_PROJECT_DIR"
+    CMAKE_PREFIX_PATH:       "/opt/rocm/lib/cmake"
+    CMAKE_HIP_COMPILER:      "/opt/rocm/llvm/bin/clang++"
+    Kokkos_CXX_COMPILER:     "/opt/rocm/llvm/bin/clang++"
+    CMAKE_HIP_ARCHITECTURES: "gfx900"
 
     # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
-    CCACHE_IGNOREOPTIONS: "-isystem=*"
-    CCACHE_COMPILERCHECK: "content"
-    CCACHE_NOHASHDIR: "true"
-    CCACHE_RESHARE: "true"
+    CCACHE_IGNOREOPTIONS:    "-isystem=*"
+    CCACHE_BASEDIR:          "$CI_PROJECT_DIR"
+    CCACHE_COMPILERCHECK:    "content"
+    CCACHE_NOHASHDIR:        "true"
+    CCACHE_RESHARE:          "true"
   after_script:
     - ccache -v -s
     - ccache -z

CMake/testing/VTKmTestWrappers.cmake

@@ -145,7 +145,7 @@ function(vtkm_unit_tests)
       list(APPEND per_device_command_line_arguments --vtkm-device=kokkos)
       list(APPEND per_device_suffix "KOKKOS")
       #may require more time because of kernel generation.
-      list(APPEND per_device_timeout 2500)
+      list(APPEND per_device_timeout 1500)
       list(APPEND per_device_serial FALSE)
     endif()
     if(VTKm_ENABLE_TBB AND (enable_all_backends OR NOT per_device_suffix))