# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Ascent.
.ascent_gcc_cuda:
  variables:
    CCACHE_BASEDIR:       "/gpfs/wolf/"
    CCACHE_DIR:           "/gpfs/wolf/csc331/scratch/vbolea/ci/ccache"
    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
    CCACHE_IGNOREOPTIONS: "-isystem=*"
    CCACHE_NOHASHDIR:     "true"

    CC:                   "gcc"
    CXX:                  "g++"
    CMAKE_BUILD_TYPE:     "RelWithDebInfo"
    CMAKE_GENERATOR:      "Ninja"
    CUDAHOSTCXX:          "g++"
    CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc331/scratch/vbolea/ci/vtk-m"

    VTKM_SETTINGS: cuda+ascent+ccache
    JOB_MODULES: >-
      git-lfs
      zstd
      cuda/11.4.2
      gcc/10.2.0
      ninja
      spectrum-mpi
      lsf-tools
      cmake
  interruptible: true

.setup_env_ecpci: &setup_env_ecpci |
  module purge
  module load ${JOB_MODULES}
  module list
  export PATH="/gpfs/wolf/csc331/scratch/vbolea/ci/utils:$PATH"

build:ascent_gcc_cuda:
  stage: build
  tags: [olcf, ascent, batch]
  extends:
    - .ascent_gcc_cuda
    - .run_ascent_ci
    - .cmake_build_artifacts
  variables:
    SCHEDULER_PARAMETERS: -P CSC331 -W 2:00 -nnodes 1 -alloc_flags smt1
  timeout: 125 minutes
  before_script:
    - *setup_env_ecpci
    - ccache -z
    - git remote add lfs https://gitlab.kitware.com/vtk/vtk-m.git
    - git fetch lfs
    - git-lfs install
    - git-lfs pull lfs
  script:
    # Each Ascent (Summit) node has 172 threads (43 cores). SMT1 is needed to
    # avoid L1 cache pollution among different processes. Thus, using 40 cores
    # seems a reasonable choice which leaves a couple of cores for system processes.
    - CTEST_MAX_PARALLELISM=40 cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake
    - ctest -VV -S .gitlab/ci/ctest_configure.cmake
    - GITLAB_CI_EMULATION=1 jsrun -n1 -a1 -g1 -c40 -bpacked:40 ctest -VV -S .gitlab/ci/ctest_build.cmake
  after_script:
    - *setup_env_ecpci
    - ccache -s
    - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake

test:ascent_gcc_cuda:
  stage: test
  tags: [olcf, ascent, batch]
  extends:
    - .ascent_gcc_cuda
    - .run_ascent_ci
    - .cmake_test_artifacts
  needs:
    - build:ascent_gcc_cuda
  dependencies:
    - build:ascent_gcc_cuda
  variables:
    # For tests we want to use a small number of proccesses, for some reason
    # a higher parallelism number tend to results in test malfunctions.
    CTEST_MAX_PARALLELISM: 4
    # We need this to skip ctest_submit from being run inside a jsrun job
    GITLAB_CI_EMULATION: 1
    SCHEDULER_PARAMETERS: -P CSC331 -W 1:00 -nnodes 1 -alloc_flags gpudefault
  timeout: 65 minutes
  before_script:
    - *setup_env_ecpci
  script:
    - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake
    - jsrun -n1 -a1 -g1 -c1 ctest -VV -S .gitlab/ci/ctest_test.cmake
  after_script:
    - *setup_env_ecpci
    - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake