From 98a1cc75141b7a8a879dee253d3f0f68d342e3d8 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Wed, 13 Oct 2021 15:40:52 -0400 Subject: [PATCH] CI: add HIP builing build - It also adds CTEST_MAX_PARALLELISM to further control the number of parallelism level while building vtk-m. Signed-off-by: Vicente Adolfo Bolea Sanchez --- .gitlab-ci.yml | 17 +++++++- .gitlab/ci/config/gitlab_ci_setup.cmake | 18 +++++++- .gitlab/ci/config/initial_config.cmake | 19 +++++++- .../docker/ubuntu2004/kokkos-hip/Dockerfile | 43 +++++++++++++++++++ .../kokkos-hip/kokkos_cmake_config.cmake | 21 +++++++++ .gitlab/ci/ubuntu2004.yml | 35 +++++++++++++++ 6 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 .gitlab/ci/docker/ubuntu2004/kokkos-hip/Dockerfile create mode 100644 .gitlab/ci/docker/ubuntu2004/kokkos-hip/kokkos_cmake_config.cmake diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 55467aa71..659fc06a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,12 +105,27 @@ extends: - .docker_image +.ubuntu2004_hip_kokkos: &ubuntu2004_hip_kokkos + image: "kitware/vtkm:ci-ubuntu2004_hip_kokkos-20210827" + extends: + - .docker_image + .only-default: &only-default only: - master@vtk/vtk-m - tags@vtk/vtk-m - merge_requests +.only-scheduled: &only-scheduled + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + allow_failure: true + - if: '$CI_MERGE_REQUEST_ID' + when: manual + allow_failure: true + - when: never + .only-master: &only-master only: - master@vtk/vtk-m @@ -189,7 +204,7 @@ stages: before_script: - *install_cmake script: - - "ctest -VV -S .gitlab/ci/ctest_test.cmake" + - "ctest $CTEST_TIMEOUT -VV -S .gitlab/ci/ctest_test.cmake" artifacts: expire_in: 24 hours when: always diff --git a/.gitlab/ci/config/gitlab_ci_setup.cmake b/.gitlab/ci/config/gitlab_ci_setup.cmake index 42a44b8a7..a6ce19e03 100644 --- a/.gitlab/ci/config/gitlab_ci_setup.cmake +++ b/.gitlab/ci/config/gitlab_ci_setup.cmake @@ -63,11 +63,25 @@ if("$ENV{VTKM_CI_NIGHTLY}" STREQUAL "TRUE") set(CTEST_TRACK "Nightly") endif() -if (CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles") +# In Make, default parallelism to number of cores. +if(CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles") include(ProcessorCount) ProcessorCount(nproc) + + if(DEFINED ENV{CTEST_MAX_PARALLELISM}) + if(nproc GREATER $ENV{CTEST_MAX_PARALLELISM}) + set(nproc $ENV{CTEST_MAX_PARALLELISM}) + endif() + endif() + set(CTEST_BUILD_FLAGS "-j${nproc}") -endif () +endif() + +# In Ninja, we do not need to specify parallelism unless we need to restrict +# the number of threads. +if(CTEST_CMAKE_GENERATOR STREQUAL "Ninja" AND DEFINED ENV{CTEST_MAX_PARALLELISM}) + set(CTEST_BUILD_FLAGS "-j$ENV{CTEST_MAX_PARALLELISM}") +endif() if(DEFINED ENV{CTEST_MEMORYCHECK_TYPE}) set(env_value "$ENV{CTEST_MEMORYCHECK_TYPE}") diff --git a/.gitlab/ci/config/initial_config.cmake b/.gitlab/ci/config/initial_config.cmake index 912978167..29d4639ac 100644 --- a/.gitlab/ci/config/initial_config.cmake +++ b/.gitlab/ci/config/initial_config.cmake @@ -56,6 +56,9 @@ foreach(option IN LISTS options) elseif(no_virtual STREQUAL option) set(VTKm_NO_DEPRECATED_VIRTUAL "ON" CACHE STRING "") + elseif(no_testing STREQUAL option) + set(VTKm_ENABLE_TESTING OFF CACHE BOOL "") + elseif(examples STREQUAL option) set(VTKm_ENABLE_EXAMPLES "ON" CACHE STRING "") @@ -64,7 +67,7 @@ foreach(option IN LISTS options) elseif(benchmarks STREQUAL option) set(VTKm_ENABLE_BENCHMARKS "ON" CACHE STRING "") - set(ENV{CMAKE_PREFIX_PATH} "$ENV{HOME}/gbench") + set(ENV{CMAKE_PREFIX_PATH} "$ENV{CMAKE_PREFIX_PATH}:$ENV{HOME}/gbench") elseif(mpi STREQUAL option) set(VTKm_ENABLE_MPI "ON" CACHE STRING "") @@ -95,6 +98,16 @@ foreach(option IN LISTS options) elseif(turing STREQUAL option) set(VTKm_CUDA_Architecture "turing" CACHE STRING "") + + elseif(hip STREQUAL option) + if(CMAKE_VERSION VERSION_LESS_EQUAL 3.20) + message(FATAL_ERROR "VTK-m requires cmake > 3.20 to enable HIP support") + endif() + + set(CMAKE_C_COMPILER "/opt/rocm/llvm/bin/clang" CACHE FILEPATH "") + set(CMAKE_CXX_COMPILER "/opt/rocm/llvm/bin/clang++" CACHE FILEPATH "") + set(VTKm_ENABLE_KOKKOS_HIP ON CACHE STRING "") + set(CMAKE_HIP_ARCHITECTURES "gfx900" CACHE STRING "") endif() endforeach() @@ -109,6 +122,10 @@ if(SCCACHE_COMMAND) set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") + if(DEFINED VTKm_ENABLE_KOKKOS_HIP) + set(CMAKE_HIP_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") + endif() + # Use VTKm_CUDA_Architecture to determine if we need CUDA sccache setup # since this will also capture when kokkos is being used with CUDA backing if(DEFINED VTKm_CUDA_Architecture) diff --git a/.gitlab/ci/docker/ubuntu2004/kokkos-hip/Dockerfile b/.gitlab/ci/docker/ubuntu2004/kokkos-hip/Dockerfile new file mode 100644 index 000000000..03521a2ba --- /dev/null +++ b/.gitlab/ci/docker/ubuntu2004/kokkos-hip/Dockerfile @@ -0,0 +1,43 @@ +FROM rocm/dev-ubuntu-20.04 +LABEL maintainer "Vicente Adolfo Bolea Sanchez" + +# Base dependencies for building VTK-m projects +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cmake \ + curl \ + g++ \ + git \ + git-lfs \ + libmpich-dev \ + libomp-dev \ + mpich \ + ninja-build \ + rsync \ + ssh \ + software-properties-common + +# Need to run git-lfs install manually on ubuntu based images when using the +# system packaged version +RUN git-lfs install + +# Provide CMake +ARG CMAKE_VERSION=3.21.1 +RUN mkdir /opt/cmake/ && \ + curl -L https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-Linux-x86_64.sh > cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ + sh cmake-$CMAKE_VERSION-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \ + rm cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ + ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest + +ENV PATH "/opt/cmake/bin:${PATH}" +ENV CMAKE_PREFIX_PATH "/opt/rocm/lib/cmake:/opt/rocm/lib:${CMAKE_PREFIX_PATH}" +ENV CMAKE_GENERATOR "Ninja" + +# Build and install Kokkos +ARG KOKKOS_VERSION=3.4.01 +COPY kokkos_cmake_config.cmake kokkos_cmake_config.cmake +RUN curl -L https://github.com/kokkos/kokkos/archive/refs/tags/$KOKKOS_VERSION.tar.gz | tar -xzf - && \ + cmake -S kokkos-$KOKKOS_VERSION -B build -C kokkos_cmake_config.cmake && \ + cmake --build build -v && \ + sudo cmake --install build + +RUN rm -rf build diff --git a/.gitlab/ci/docker/ubuntu2004/kokkos-hip/kokkos_cmake_config.cmake b/.gitlab/ci/docker/ubuntu2004/kokkos-hip/kokkos_cmake_config.cmake new file mode 100644 index 000000000..7b71f7b65 --- /dev/null +++ b/.gitlab/ci/docker/ubuntu2004/kokkos-hip/kokkos_cmake_config.cmake @@ -0,0 +1,21 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +set(CMAKE_BUILD_TYPE "release" CACHE STRING "") +set(CMAKE_INSTALL_PREFIX /opt/kokkos CACHE PATH "") +set(CMAKE_C_COMPILER /opt/rocm/llvm/bin/clang CACHE FILEPATH "") +set(CMAKE_CXX_COMPILER /opt/rocm/llvm/bin/clang++ CACHE FILEPATH "") +set(CMAKE_CXX_STANDARD "14" CACHE STRING "") +set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "") + +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") +set(Kokkos_ARCH_VEGA900 ON CACHE BOOL "") +set(Kokkos_ENABLE_HIP ON CACHE BOOL "") +set(Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE OFF CACHE BOOL "") diff --git a/.gitlab/ci/ubuntu2004.yml b/.gitlab/ci/ubuntu2004.yml index bed966203..23528089f 100644 --- a/.gitlab/ci/ubuntu2004.yml +++ b/.gitlab/ci/ubuntu2004.yml @@ -59,3 +59,38 @@ test:ubuntu2004_kokkos: - build:ubuntu2004_kokkos needs: - build:ubuntu2004_kokkos + +build:ubuntu2004_hip_kokkos: + tags: + - build + - vtkm + - docker + - radeon + extends: + - .ubuntu2004_hip_kokkos + - .cmake_build_linux + - .only-scheduled + variables: + CMAKE_BUILD_TYPE: RelWithDebInfo + VTKM_SETTINGS: "benchmarks+kokkos+hip+no_virtual+no_rendering" + CMAKE_PREFIX_PATH: "/opt/rocm/lib/cmake" + CTEST_MAX_PARALLELISM: "3" + timeout: 12 hours + +test:ubuntu2004_hip_kokkos: + tags: + - build + - vtkm + - docker + - radeon + extends: + - .ubuntu2004_hip_kokkos + - .cmake_test_linux + - .only-default + variables: + CTEST_TIMEOUT: "30" + dependencies: + - build:ubuntu2004_hip_kokkos + needs: + - build:ubuntu2004_hip_kokkos + timeout: 2 hours