From a43717de9e30daceeb8066b27dcea5577148d923 Mon Sep 17 00:00:00 2001
From: Vicente Adolfo Bolea Sanchez <vicente.bolea@kitware.com>
Date: Tue, 9 Jul 2024 21:06:27 -0400
Subject: [PATCH] ci: add frontier nightly build

---
 .gitlab-ci.yml                           |  6 ++--
 .gitlab/ci/{crusher.yml => frontier.yml} | 39 +++++++++++++-----------
 2 files changed, 24 insertions(+), 21 deletions(-)
 rename .gitlab/ci/{crusher.yml => frontier.yml} (75%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87c935432..3ff62cd54 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,9 +107,9 @@
       when: on_success
     - when: never
 
-.run_crusher_ci: &run_crusher_ci
+.run_frontier_ci: &run_frontier_ci
   rules:
-    - if: '$CI_PROJECT_PATH == "ci/csc331_crusher/dev/vtk-m"'
+    - if: '$CI_PROJECT_PATH == "ci/ums033/dev/vtk-m"'
       when: on_success
     - when: never
 
@@ -235,10 +235,10 @@ stages:
 
 
 include:
-  - local: '/.gitlab/ci/crusher.yml'
   - local: '/.gitlab/ci/centos7.yml'
   - local: '/.gitlab/ci/centos8.yml'
   - local: '/.gitlab/ci/doxygen.yml'
+  - local: '/.gitlab/ci/frontier.yml'
   - local: '/.gitlab/ci/macos.yml'
   - local: '/.gitlab/ci/opensuse.yml'
   - local: '/.gitlab/ci/rhel8.yml'
diff --git a/.gitlab/ci/crusher.yml b/.gitlab/ci/frontier.yml
similarity index 75%
rename from .gitlab/ci/crusher.yml
rename to .gitlab/ci/frontier.yml
index de029b56e..936f6283f 100644
--- a/.gitlab/ci/crusher.yml
+++ b/.gitlab/ci/frontier.yml
@@ -11,11 +11,11 @@
 ##=============================================================================
 
 # Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Spock.
-.crusher_gcc_hip:
+.frontier_gcc_hip:
   variables:
-    CCACHE_BASEDIR:          "/lustre/orion/csc331/scratch/"
-    CCACHE_DIR:              "/lustre/orion/csc331/scratch/vbolea/ci/vtk-m/ccache"
-    CUSTOM_CI_BUILDS_DIR:    "/lustre/orion/csc331/scratch/vbolea/ci/vtk-m/runtime"
+    CCACHE_BASEDIR:          "/lustre/orion/ums033/scratch/"
+    CCACHE_DIR:              "/lustre/orion/ums033/scratch/ums033_auser/ci/vtk-m/ccache"
+    CUSTOM_CI_BUILDS_DIR:    "/lustre/orion/ums033/scratch/ums033_auser/ci/vtk-m/runtime"
 
     # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
     CCACHE_IGNOREOPTIONS:    "-isystem=*"
@@ -45,7 +45,8 @@
       git-lfs
       ninja
       zstd
-    VTKM_SETTINGS: kokkos+hip+gfx90a+crusher+ccache+no_rendering
+    OLCF_SERVICE_ACCOUNT: "ums033_auser"
+    VTKM_SETTINGS: kokkos+hip+gfx90a+frontier+ccache+no_rendering
   interruptible: true
 
 .setup_env_ecpci: &setup_env_ecpci |
@@ -54,12 +55,12 @@
   module list
   export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"
 
-build:crusher_gcc_hip:
+build:frontier_gcc_hip:
   stage: build
-  tags: [crusher, shell]
+  tags: [frontier, shell]
   extends:
-    - .crusher_gcc_hip
-    - .run_crusher_ci
+    - .frontier_gcc_hip
+    - .run_frontier_ci
   before_script:
     - *setup_env_ecpci
     - mkdir -p "$CCACHE_INSTALL_DIR"
@@ -83,18 +84,20 @@ build:crusher_gcc_hip:
       - build/
       - .gitlab/ccache/ccache
 
-test:crusher_gcc_hip:
+test:frontier_gcc_hip:
   stage: test
-  tags: [crusher, slurm]
+  tags: [frontier, slurm]
   extends:
-    - .crusher_gcc_hip
-    - .run_crusher_ci
+    - .frontier_gcc_hip
+    - .run_frontier_ci
   needs:
-    - build:crusher_gcc_hip
+    - build:frontier_gcc_hip
   dependencies:
-    - build:crusher_gcc_hip
+    - build:frontier_gcc_hip
   variables:
-    SCHEDULER_PARAMETERS: "-ACSC331_crusher -pbatch -t120 --nice=0 -c32 --gpus=4 -N 1"
+    # -c48 --gpus=8 since we want to use fully utilize a node
+    # SMT=2 since compiling is balanced type of task (IO|computing).
+    SCHEDULER_PARAMETERS: "-AUMS033 -pbatch -t 00:30:00 --nice=0 -c48 --threads-per-core=2 --gpus=8 -N 1"
     # We need this to skip ctest_submit from being run inside a jsrun job
     GITLAB_CI_EMULATION: 1
     # Tests errors to address due to different env in Spock
@@ -107,8 +110,8 @@ test:crusher_gcc_hip:
   before_script:
     - *setup_env_ecpci
   script:
-    - CTEST_MAX_PARALLELISM=32 ctest -VV -S .gitlab/ci/ctest_build.cmake
-    - CTEST_MAX_PARALLELISM=4  ctest -VV -S .gitlab/ci/ctest_test.cmake
+    - CTEST_MAX_PARALLELISM=96 ctest -VV -S .gitlab/ci/ctest_build.cmake
+    - CTEST_MAX_PARALLELISM=8  ctest -VV -S .gitlab/ci/ctest_test.cmake
   after_script:
     - *setup_env_ecpci
     - ccache -s