From 2d1b609b34d6b0f6f6e0eac72dc133ab14a342d8 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 12 Aug 2020 13:56:48 -0400 Subject: [PATCH] Use Ubuntu instead of rhel8 for cuda+kokkos The rhel8 image would never upload to gitlab-ci completed artifacts. --- .gitlab-ci.yml | 10 ++-- .gitlab/ci/config/initial_config.cmake | 5 +- .gitlab/ci/docker/rhel8/kokkos/Dockerfile | 30 ------------ .../docker/ubuntu1804/kokkos-cuda/Dockerfile | 47 +++++++++++++++++++ .gitlab/ci/docker/update_all.sh | 8 ++-- .gitlab/ci/rhel8.yml | 37 --------------- .gitlab/ci/ubuntu1804.yml | 38 +++++++++++++++ 7 files changed, 98 insertions(+), 77 deletions(-) delete mode 100644 .gitlab/ci/docker/rhel8/kokkos/Dockerfile create mode 100644 .gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 429b8ade9..43d6fcccb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,11 +63,6 @@ extends: - .docker_image -.rhel8_kokkos: &rhel8_kokkos - image: "kitware/vtkm:ci-rhel8_kokkos-20200729" - extends: - - .docker_image - .ubuntu1604: &ubuntu1604 image: "kitware/vtkm:ci-ubuntu1604-20200729" extends: @@ -88,6 +83,11 @@ extends: - .docker_image +.ubuntu1804_cuda_kokkos: &ubuntu1804_cuda_kokkos + image: "kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-20200729" + extends: + - .docker_image + .ubuntu2004_doxygen: &ubuntu2004_doxygen image: "kitware/vtkm:ci-doxygen-20200729" extends: diff --git a/.gitlab/ci/config/initial_config.cmake b/.gitlab/ci/config/initial_config.cmake index 0f75cbe4a..e4df50de0 100644 --- a/.gitlab/ci/config/initial_config.cmake +++ b/.gitlab/ci/config/initial_config.cmake @@ -97,7 +97,10 @@ find_program(SCCACHE_COMMAND NAMES sccache) if(SCCACHE_COMMAND) set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") - if(VTKm_ENABLE_CUDA) + + # Use VTKm_CUDA_Architecture to determine if we need CUDA sccache setup + # since this will also capture when kokkos is being used with CUDA backing + if(DEFINED VTKm_CUDA_Architecture) set(CMAKE_CUDA_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "") endif() endif() diff --git a/.gitlab/ci/docker/rhel8/kokkos/Dockerfile b/.gitlab/ci/docker/rhel8/kokkos/Dockerfile deleted file mode 100644 index ff59b9fce..000000000 --- a/.gitlab/ci/docker/rhel8/kokkos/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM nvidia/cuda:10.2-devel-ubi8 -LABEL maintainer "Sujin Philip" - -RUN yum install make gcc gcc-c++ curl -y -RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash -RUN yum install git git-lfs -y - -# kokkos backend requires cmake 3.18 -RUN mkdir /opt/cmake/ && \ - curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0-Linux-x86_64.sh > cmake-3.18.0-Linux-x86_64.sh && \ - sh cmake-3.18.0-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \ - rm cmake-3.18.0-Linux-x86_64.sh && \ - ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest - -ENV PATH "/opt/cmake/bin:${PATH}" - -# Build and install Kokkos -RUN mkdir -p /opt/kokkos/build && \ - cd /opt/kokkos/build && \ - curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \ - tar -xf kokkos-3.1.01.tar.gz && \ - mkdir bld && cd bld && \ - CXX=/opt/kokkos/build/kokkos-3.1.01/bin/nvcc_wrapper \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC \ - -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_CONSTEXPR=ON \ - -DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_CUDA_LDG_INTRINSIC=ON \ - -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON -DKokkos_ENABLE_CUDA_UVM=ON \ - -DKokkos_ARCH_TURING75=ON ../kokkos-3.1.01 && \ - make -j all && \ - make install diff --git a/.gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile b/.gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile new file mode 100644 index 000000000..6d070ed1c --- /dev/null +++ b/.gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile @@ -0,0 +1,47 @@ +FROM nvidia/cuda:11.0-devel-ubuntu18.04 +LABEL maintainer "Robert Maynard" + +# Base dependencies for building VTK-m projects +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + g++ \ + git \ + git-lfs \ + ninja-build \ + && \ + rm -rf /var/lib/apt/lists/* + +# Need to run git-lfs install manually on ubuntu based images when using the +# system packaged version +RUN git-lfs install + +# kokkos backend requires cmake 3.18 +RUN mkdir /opt/cmake/ && \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.18.1/cmake-3.18.1-Linux-x86_64.sh > cmake-3.18.1-Linux-x86_64.sh && \ + sh cmake-3.18.1-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \ + rm cmake-3.18.1-Linux-x86_64.sh && \ + ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest + +ENV PATH "/opt/cmake/bin:${PATH}" + +# Build and install Kokkos +RUN mkdir -p /opt/kokkos/build && \ + cd /opt/kokkos/build && \ + curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \ + tar -xf kokkos-3.1.01.tar.gz && \ + mkdir bld && cd bld && \ + CXX=/opt/kokkos/build/kokkos-3.1.01/bin/nvcc_wrapper \ + cmake -B . -S ../kokkos-3.1.01 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/kokkos \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_CXX_STANDARD=14 \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ENABLE_CUDA_CONSTEXPR=ON \ + -DKokkos_ENABLE_CUDA_LAMBDA=ON \ + -DKokkos_ENABLE_CUDA_LDG_INTRINSIC=ON \ + -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \ + -DKokkos_ENABLE_CUDA_UVM=ON \ + -DKokkos_ARCH_TURING75=ON && \ + cmake --build . -j 8 && \ + cmake --install . diff --git a/.gitlab/ci/docker/update_all.sh b/.gitlab/ci/docker/update_all.sh index 2231a2df4..644061af5 100755 --- a/.gitlab/ci/docker/update_all.sh +++ b/.gitlab/ci/docker/update_all.sh @@ -18,10 +18,6 @@ cd rhel8/cuda10.2 sudo docker build -t kitware/vtkm:ci-rhel8_cuda10.2-$date . cd ../.. -cd rhel8/kokkos -sudo docker build -t kitware/vtkm:ci-rhel8_kokkos-$date . -cd ../.. - cd ubuntu1604/base sudo docker build -t kitware/vtkm:ci-ubuntu1604-$date . cd ../.. @@ -38,6 +34,10 @@ cd ubuntu1804/cuda10.1 sudo docker build -t kitware/vtkm:ci-ubuntu1804_cuda10.1-$date . cd ../.. +cd ubuntu1804/kokkos-cuda +sudo docker build -t kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-$date . +cd ../.. + cd ubuntu2004/doxygen/ sudo docker build -t kitware/vtkm:ci-doxygen-$date . cd ../.. diff --git a/.gitlab/ci/rhel8.yml b/.gitlab/ci/rhel8.yml index 3733f8305..43a8a7fc7 100644 --- a/.gitlab/ci/rhel8.yml +++ b/.gitlab/ci/rhel8.yml @@ -60,40 +60,3 @@ test:rhel8_vtk_types: - build:rhel8_vtk_types needs: - build:rhel8_vtk_types - -# Build on rhel8 with kokkos and test on rhel8 -# Uses gcc 8.2.1 -build:rhel8_kokkos: - tags: - - build - - vtkm - - docker - - linux - - large-memory - extends: - - .rhel8_kokkos - - .cmake_build_linux - - .only-default - variables: - CMAKE_GENERATOR: "Unix Makefiles" - CMAKE_BUILD_TYPE: Release - VTKM_SETTINGS: "kokkos+static+64bit_floats" - -test:rhel8_kokkos: - tags: - - test - - cuda-rt - - turing - - vtkm - - docker - - linux - extends: - - .rhel8_kokkos - - .cmake_test_linux - - .only-default - dependencies: - - build:rhel8_kokkos - needs: - - build:rhel8_kokkos - variables: - CUDA_LAUNCH_BLOCKING: "1" diff --git a/.gitlab/ci/ubuntu1804.yml b/.gitlab/ci/ubuntu1804.yml index 9cdccb12e..e9b4de1e7 100644 --- a/.gitlab/ci/ubuntu1804.yml +++ b/.gitlab/ci/ubuntu1804.yml @@ -180,3 +180,41 @@ test:ubuntu1804_clang8: - build:ubuntu1804_clang8 needs: - build:ubuntu1804_clang8 + +# Build on ubuntu1804 with kokkos and test on ubuntu1804 +# Uses CUDA 11 +build:ubuntu1804_kokkos: + tags: + - build + - vtkm + - docker + - linux + - cuda-rt + - large-memory + extends: + - .ubuntu1804_cuda_kokkos + - .cmake_build_linux + - .only-default + variables: + CMAKE_GENERATOR: "Ninja" + CMAKE_BUILD_TYPE: Release + VTKM_SETTINGS: "kokkos+static+64bit_floats" + +test:ubuntu1804_kokkos: + tags: + - test + - vtkm + - docker + - linux + - cuda-rt + - turing + extends: + - .ubuntu1804_cuda_kokkos + - .cmake_test_linux + - .only-default + dependencies: + - build:ubuntu1804_kokkos + needs: + - build:ubuntu1804_kokkos + variables: + CUDA_LAUNCH_BLOCKING: "1"