Use Ubuntu instead of rhel8 for cuda+kokkos

The rhel8 image would never upload to gitlab-ci completed artifacts.
2020-08-12 13:56:48 -04:00 · 2020-08-12 13:56:48 -04:00 · 2d1b609b34
commit 2d1b609b34
parent 7692485831
7 changed files with 98 additions and 77 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -63,11 +63,6 @@
  extends:
    - .docker_image

-.rhel8_kokkos: &rhel8_kokkos
-  image: "kitware/vtkm:ci-rhel8_kokkos-20200729"
-  extends:
-    - .docker_image
-
 .ubuntu1604: &ubuntu1604
  image: "kitware/vtkm:ci-ubuntu1604-20200729"
  extends:
@ -88,6 +83,11 @@
  extends:
    - .docker_image

+.ubuntu1804_cuda_kokkos: &ubuntu1804_cuda_kokkos
+  image: "kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-20200729"
+  extends:
+    - .docker_image
+
 .ubuntu2004_doxygen: &ubuntu2004_doxygen
  image: "kitware/vtkm:ci-doxygen-20200729"
  extends:
--- a/.gitlab/ci/config/initial_config.cmake
+++ b/.gitlab/ci/config/initial_config.cmake
@ -97,7 +97,10 @@ find_program(SCCACHE_COMMAND NAMES sccache)
 if(SCCACHE_COMMAND)
  set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
  set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
-  if(VTKm_ENABLE_CUDA)
+
+  # Use VTKm_CUDA_Architecture to determine if we need CUDA sccache setup
+  # since this will also capture when kokkos is being used with CUDA backing
+  if(DEFINED VTKm_CUDA_Architecture)
    set(CMAKE_CUDA_COMPILER_LAUNCHER "${SCCACHE_COMMAND}" CACHE STRING "")
  endif()
 endif()
--- a/.gitlab/ci/docker/rhel8/kokkos/Dockerfile
+++ b/.gitlab/ci/docker/rhel8/kokkos/Dockerfile
@ -1,30 +0,0 @@
-FROM nvidia/cuda:10.2-devel-ubi8
-LABEL maintainer "Sujin Philip<sujin.philip@kitware.com>"
-
-RUN yum install make gcc gcc-c++ curl -y
-RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash
-RUN yum install git git-lfs -y
-
-# kokkos backend requires cmake 3.18
-RUN mkdir /opt/cmake/ && \
-    curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0-Linux-x86_64.sh > cmake-3.18.0-Linux-x86_64.sh && \
-    sh cmake-3.18.0-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \
-    rm cmake-3.18.0-Linux-x86_64.sh && \
-    ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest
-
-ENV PATH "/opt/cmake/bin:${PATH}"
-
-# Build and install Kokkos
-RUN mkdir -p /opt/kokkos/build && \
-    cd /opt/kokkos/build && \
-    curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \
-    tar -xf kokkos-3.1.01.tar.gz && \
-    mkdir bld && cd bld && \
-    CXX=/opt/kokkos/build/kokkos-3.1.01/bin/nvcc_wrapper \
-    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC \
-          -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_CONSTEXPR=ON \
-          -DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_CUDA_LDG_INTRINSIC=ON \
-          -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON -DKokkos_ENABLE_CUDA_UVM=ON \
-          -DKokkos_ARCH_TURING75=ON ../kokkos-3.1.01 && \
-    make -j all && \
-    make install
--- a/.gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile
+++ b/.gitlab/ci/docker/ubuntu1804/kokkos-cuda/Dockerfile
@ -0,0 +1,47 @@
+FROM nvidia/cuda:11.0-devel-ubuntu18.04
+LABEL maintainer "Robert Maynard<robert.maynard@kitware.com>"
+
+# Base dependencies for building VTK-m projects
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      curl \
+      g++ \
+      git \
+      git-lfs \
+      ninja-build \
+      && \
+    rm -rf /var/lib/apt/lists/*
+
+# Need to run git-lfs install manually on ubuntu based images when using the
+# system packaged version
+RUN git-lfs install
+
+# kokkos backend requires cmake 3.18
+RUN mkdir /opt/cmake/ && \
+    curl -L https://github.com/Kitware/CMake/releases/download/v3.18.1/cmake-3.18.1-Linux-x86_64.sh > cmake-3.18.1-Linux-x86_64.sh && \
+    sh cmake-3.18.1-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \
+    rm cmake-3.18.1-Linux-x86_64.sh && \
+    ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest
+
+ENV PATH "/opt/cmake/bin:${PATH}"
+
+# Build and install Kokkos
+RUN mkdir -p /opt/kokkos/build && \
+    cd /opt/kokkos/build && \
+    curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \
+    tar -xf kokkos-3.1.01.tar.gz && \
+    mkdir bld && cd bld && \
+    CXX=/opt/kokkos/build/kokkos-3.1.01/bin/nvcc_wrapper \
+    cmake -B . -S ../kokkos-3.1.01 \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=/opt/kokkos \
+          -DCMAKE_CXX_FLAGS=-fPIC \
+          -DCMAKE_CXX_STANDARD=14 \
+          -DKokkos_ENABLE_CUDA=ON \
+          -DKokkos_ENABLE_CUDA_CONSTEXPR=ON \
+          -DKokkos_ENABLE_CUDA_LAMBDA=ON \
+          -DKokkos_ENABLE_CUDA_LDG_INTRINSIC=ON \
+          -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \
+          -DKokkos_ENABLE_CUDA_UVM=ON \
+          -DKokkos_ARCH_TURING75=ON && \
+    cmake --build . -j 8 && \
+    cmake --install .
--- a/.gitlab/ci/docker/update_all.sh
+++ b/.gitlab/ci/docker/update_all.sh
@ -18,10 +18,6 @@ cd rhel8/cuda10.2
 sudo docker build -t kitware/vtkm:ci-rhel8_cuda10.2-$date .
 cd ../..

-cd rhel8/kokkos
-sudo docker build -t kitware/vtkm:ci-rhel8_kokkos-$date .
-cd ../..
-
 cd ubuntu1604/base
 sudo docker build -t kitware/vtkm:ci-ubuntu1604-$date .
 cd ../..
@ -38,6 +34,10 @@ cd ubuntu1804/cuda10.1
 sudo docker build -t kitware/vtkm:ci-ubuntu1804_cuda10.1-$date .
 cd ../..

+cd ubuntu1804/kokkos-cuda
+sudo docker build -t kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-$date .
+cd ../..
+
 cd ubuntu2004/doxygen/
 sudo docker build -t kitware/vtkm:ci-doxygen-$date .
 cd ../..
--- a/.gitlab/ci/rhel8.yml
+++ b/.gitlab/ci/rhel8.yml
@ -60,40 +60,3 @@ test:rhel8_vtk_types:
    - build:rhel8_vtk_types
  needs:
    - build:rhel8_vtk_types
-
-# Build on rhel8 with kokkos and test on rhel8
-# Uses gcc 8.2.1
-build:rhel8_kokkos:
-  tags:
-    - build
-    - vtkm
-    - docker
-    - linux
-    - large-memory
-  extends:
-    - .rhel8_kokkos
-    - .cmake_build_linux
-    - .only-default
-  variables:
-    CMAKE_GENERATOR: "Unix Makefiles"
-    CMAKE_BUILD_TYPE: Release
-    VTKM_SETTINGS: "kokkos+static+64bit_floats"
-
-test:rhel8_kokkos:
-  tags:
-    - test
-    - cuda-rt
-    - turing
-    - vtkm
-    - docker
-    - linux
-  extends:
-    - .rhel8_kokkos
-    - .cmake_test_linux
-    - .only-default
-  dependencies:
-    - build:rhel8_kokkos
-  needs:
-    - build:rhel8_kokkos
-  variables:
-    CUDA_LAUNCH_BLOCKING: "1"
--- a/.gitlab/ci/ubuntu1804.yml
+++ b/.gitlab/ci/ubuntu1804.yml
@ -180,3 +180,41 @@ test:ubuntu1804_clang8:
    - build:ubuntu1804_clang8
  needs:
    - build:ubuntu1804_clang8
+
+# Build on ubuntu1804 with kokkos and test on ubuntu1804
+# Uses CUDA 11
+build:ubuntu1804_kokkos:
+  tags:
+    - build
+    - vtkm
+    - docker
+    - linux
+    - cuda-rt
+    - large-memory
+  extends:
+    - .ubuntu1804_cuda_kokkos
+    - .cmake_build_linux
+    - .only-default
+  variables:
+    CMAKE_GENERATOR: "Ninja"
+    CMAKE_BUILD_TYPE: Release
+    VTKM_SETTINGS: "kokkos+static+64bit_floats"
+
+test:ubuntu1804_kokkos:
+  tags:
+    - test
+    - vtkm
+    - docker
+    - linux
+    - cuda-rt
+    - turing
+  extends:
+    - .ubuntu1804_cuda_kokkos
+    - .cmake_test_linux
+    - .only-default
+  dependencies:
+    - build:ubuntu1804_kokkos
+  needs:
+    - build:ubuntu1804_kokkos
+  variables:
+    CUDA_LAUNCH_BLOCKING: "1"