diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91381f6b3..5aff61d80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,6 +138,12 @@ when: on_success - when: never +.run_spock_ci: &run_spock_ci + rules: + - if: '$CI_PROJECT_PATH == "ci/csc331_crusher/dev/vtk-m"' + when: on_success + - when: never + # General Longer Term Tasks: # - Setup clang tidy as sub-pipeline # - Setup a machine to replicate the issue in https://gitlab.kitware.com/vtk/vtk-m/-/issues/447 @@ -250,6 +256,7 @@ stages: include: - local: '/.gitlab/ci/ascent.yml' + - local: '/.gitlab/ci/spock.yml' - local: '/.gitlab/ci/centos7.yml' - local: '/.gitlab/ci/centos8.yml' - local: '/.gitlab/ci/doxygen.yml' diff --git a/.gitlab/ci/config/initial_config.cmake b/.gitlab/ci/config/initial_config.cmake index 6c12cfe97..89a2035fd 100644 --- a/.gitlab/ci/config/initial_config.cmake +++ b/.gitlab/ci/config/initial_config.cmake @@ -115,10 +115,7 @@ foreach(option IN LISTS options) message(FATAL_ERROR "VTK-m requires cmake > 3.20 to enable HIP support") endif() - set(CMAKE_HIP_COMPILER "/opt/rocm/llvm/bin/clang++" CACHE FILEPATH "") - set(Kokkos_CXX_COMPILER "/opt/rocm/llvm/bin/clang++" CACHE FILEPATH "") set(VTKm_ENABLE_KOKKOS_HIP ON CACHE STRING "") - set(CMAKE_HIP_ARCHITECTURES "gfx900" CACHE STRING "") # -O1 and -O2 results in ridiculous build times in ROCm < 5.3 set(CMAKE_HIP_FLAGS "-O0 " CACHE STRING "") diff --git a/.gitlab/ci/spock.yml b/.gitlab/ci/spock.yml new file mode 100644 index 000000000..7e39cfc2a --- /dev/null +++ b/.gitlab/ci/spock.yml @@ -0,0 +1,104 @@ +# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Spock. +.spock_gcc_hip: + variables: + CCACHE_BASEDIR: "/gpfs/alpine/world-shared/csc331/" + CCACHE_DIR: "/gpfs/alpine/world-shared/csc331/vtk-m/ci/ccache" + CUSTOM_CI_BUILDS_DIR: "/gpfs/alpine/world-shared/csc331/vtk-m/ci/runtime" + + # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it + CCACHE_IGNOREOPTIONS: "-isystem=*" + CCACHE_NOHASHDIR: "true" + CCACHE_INSTALL_DIR: "$CI_PROJECT_DIR/build" + + CMAKE_BUILD_TYPE: "RelWithDebInfo" + CMAKE_GENERATOR: "Ninja" + + CMAKE_HIP_COMPILER: "/opt/rocm-default/llvm/bin/clang++" + Kokkos_CXX_COMPILER: "/opt/rocm-default/llvm/bin/clang++" + CMAKE_HIP_ARCHITECTURES: "gfx908" + + CC: gcc + CXX: g++ + # DefApps/default;craype;rocm;gcc should be loaded first + JOB_MODULES: >- + DefApps/default + craype-accel-amd-gfx90a + rocm/5 + gcc/10 + cmake/3.22 + git + git-lfs + kokkos/3.5.00 + lsf-tools + ninja + spectrum-mpi + zstd + VTKM_SETTINGS: kokkos+hip+spock+ccache+no_virtual+no_rendering + interruptible: true + +.setup_env_ecpci: &setup_env_ecpci | + module purge + module load ${JOB_MODULES} + module list + export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH" + +build:spock_gcc_hip: + stage: build + tags: [spock, shell] + extends: + - .spock_gcc_hip + - .run_spock_ci + before_script: + - *setup_env_ecpci + - mkdir -p "$CCACHE_INSTALL_DIR" + - cmake --version + - cmake -VV -P .gitlab/ci/config/ccache.cmake + - ccache -z + - ccache -s + + - git remote add lfs https://gitlab.kitware.com/vtk/vtk-m.git + - git fetch lfs + - git-lfs install + - git-lfs pull lfs + script: + - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake + - ctest -VV -S .gitlab/ci/ctest_configure.cmake + artifacts: + expire_in: 24 hours + when: always + paths: + - build/ + - .gitlab/ccache/ccache + +test:spock_gcc_hip: + stage: test + tags: [spock, slurm] + extends: + - .spock_gcc_hip + - .run_spock_ci + needs: + - build:spock_gcc_hip + dependencies: + - build:spock_gcc_hip + variables: + SCHEDULER_PARAMETERS: "-ACSC331 -pecp -t120 --nice=0 -c32 --gpus=4 -N 1" + # We need this to skip ctest_submit from being run inside a jsrun job + GITLAB_CI_EMULATION: 1 + # Tests errors to address due to different env in Spock + # Refer to issue: https://gitlab.kitware.com/vtk/vtk-m/-/issues/743 + CTEST_EXCLUSIONS: >- + UnitTestMIRFilter + UnitTestDistributedBranchDecompositionFilter + UnitTestWorkletParticleAdvection + UnitTestLagrangianFilter + before_script: + - *setup_env_ecpci + script: + - CTEST_MAX_PARALLELISM=32 ctest -VV -S .gitlab/ci/ctest_build.cmake + - CTEST_MAX_PARALLELISM=4 ctest -VV -S .gitlab/ci/ctest_test.cmake + after_script: + - *setup_env_ecpci + - ccache -s + - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake + - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake + timeout: 150 minutes diff --git a/.gitlab/ci/ubuntu2004.yml b/.gitlab/ci/ubuntu2004.yml index 7409f1350..4398933fd 100644 --- a/.gitlab/ci/ubuntu2004.yml +++ b/.gitlab/ci/ubuntu2004.yml @@ -71,17 +71,20 @@ build:ubuntu2004_hip_kokkos: - .cmake_build_linux - .run_automatically variables: - CMAKE_BUILD_TYPE: RelWithDebInfo - VTKM_SETTINGS: "benchmarks+kokkos+hip+no_virtual+no_rendering+ccache" - CMAKE_PREFIX_PATH: "/opt/rocm/lib/cmake" + CMAKE_BUILD_TYPE: "RelWithDebInfo" + VTKM_SETTINGS: "benchmarks+kokkos+hip+no_virtual+no_rendering+ccache" - CCACHE_BASEDIR: "$CI_PROJECT_DIR" + CMAKE_PREFIX_PATH: "/opt/rocm/lib/cmake" + CMAKE_HIP_COMPILER: "/opt/rocm/llvm/bin/clang++" + Kokkos_CXX_COMPILER: "/opt/rocm/llvm/bin/clang++" + CMAKE_HIP_ARCHITECTURES: "gfx900" # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it - CCACHE_IGNOREOPTIONS: "-isystem=*" - CCACHE_COMPILERCHECK: "content" - CCACHE_NOHASHDIR: "true" - CCACHE_RESHARE: "true" + CCACHE_IGNOREOPTIONS: "-isystem=*" + CCACHE_BASEDIR: "$CI_PROJECT_DIR" + CCACHE_COMPILERCHECK: "content" + CCACHE_NOHASHDIR: "true" + CCACHE_RESHARE: "true" after_script: - ccache -v -s - ccache -z diff --git a/CMake/testing/VTKmTestWrappers.cmake b/CMake/testing/VTKmTestWrappers.cmake index 9af0415c9..c36ba67ad 100644 --- a/CMake/testing/VTKmTestWrappers.cmake +++ b/CMake/testing/VTKmTestWrappers.cmake @@ -145,7 +145,7 @@ function(vtkm_unit_tests) list(APPEND per_device_command_line_arguments --vtkm-device=kokkos) list(APPEND per_device_suffix "KOKKOS") #may require more time because of kernel generation. - list(APPEND per_device_timeout 2500) + list(APPEND per_device_timeout 1500) list(APPEND per_device_serial FALSE) endif() if(VTKm_ENABLE_TBB AND (enable_all_backends OR NOT per_device_suffix))