Merge topic 'update-kokkos'

447b8e711 Fill input to test of array extract from stride array
726bb0910 Fix floating point exception in Kokkos sort
89245c3df Remove NUMA regions option
4912d1d04 Update --kokkos-threads to --kokkos-num-threads
9f77e1118 Do not test if Kokkos device id does not match
fa30d6774 Update the minimum Kokkos required to 3.7
3a96e9429 ci: update Kokkos docker images
674572419 Use Kokkos 3.7.1  in the CI builds

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Vicente Bolea <vicente.bolea@kitware.com>
Merge-request: !2967
This commit is contained in:
Kenneth Moreland 2023-02-01 19:46:05 +00:00 committed by Kitware Robot
commit ce5ac9bfb0
19 changed files with 49 additions and 140 deletions

@ -38,7 +38,7 @@
- .docker_image
.ubuntu1804_cuda_kokkos: &ubuntu1804_cuda_kokkos
image: "kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-20220407"
image: "kitware/vtkm:ci-ubuntu1804_cuda11_kokkos-20230125"
extends:
- .docker_image
@ -53,12 +53,12 @@
- .docker_image
.ubuntu2004_kokkos: &ubuntu2004_kokkos
image: "kitware/vtkm:ci-ubuntu2004_kokkos-20210916"
image: "kitware/vtkm:ci-ubuntu2004_kokkos-20230125"
extends:
- .docker_image
.ubuntu2004_hip_kokkos: &ubuntu2004_hip_kokkos
image: "kitware/vtkm:ci-ubuntu2004_hip_kokkos-20220620"
image: "kitware/vtkm:ci-ubuntu2004_hip_kokkos-20230125"
extends:
- .docker_image

@ -25,13 +25,14 @@ RUN mkdir /opt/cmake/ && \
ENV PATH "/opt/cmake/bin:${PATH}"
# Build and install Kokkos
ARG KOKKOS_VERSION=3.7.01
RUN mkdir -p /opt/kokkos/build && \
cd /opt/kokkos/build && \
curl -L https://github.com/kokkos/kokkos/archive/refs/tags/3.4.01.tar.gz > kokkos-3.4.01.tar.gz && \
tar -xf kokkos-3.4.01.tar.gz && \
curl -L https://github.com/kokkos/kokkos/archive/refs/tags/$KOKKOS_VERSION.tar.gz > kokkos-$KOKKOS_VERSION.tar.gz && \
tar -xf kokkos-$KOKKOS_VERSION.tar.gz && \
mkdir bld && cd bld && \
CXX=/opt/kokkos/build/kokkos-3.4.01/bin/nvcc_wrapper \
cmake -B . -S ../kokkos-3.4.01 \
CXX=/opt/kokkos/build/kokkos-$KOKKOS_VERSION/bin/nvcc_wrapper \
cmake -B . -S ../kokkos-$KOKKOS_VERSION \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/opt/kokkos \
-DCMAKE_CXX_FLAGS=-fPIC \

@ -44,7 +44,7 @@ ENV CMAKE_PREFIX_PATH "/opt/rocm/lib/cmake:/opt/rocm/lib:${CMAKE_PREFIX_PATH}"
ENV CMAKE_GENERATOR "Ninja"
# Build and install Kokkos
ARG KOKKOS_VERSION=3.6.00
ARG KOKKOS_VERSION=3.7.01
COPY kokkos_cmake_config.cmake kokkos_cmake_config.cmake
RUN curl -L https://github.com/kokkos/kokkos/archive/refs/tags/$KOKKOS_VERSION.tar.gz | tar -xzf - && \
cmake -S kokkos-$KOKKOS_VERSION -B build -C kokkos_cmake_config.cmake && \

@ -31,11 +31,12 @@ RUN mkdir /opt/cmake/ && \
ENV PATH "${PATH}:/opt/cmake/bin"
# Build and install Kokkos
ARG KOKKOS_VERSION=3.7.01
RUN mkdir -p /opt/kokkos/build && \
cd /opt/kokkos/build && \
curl -L https://github.com/kokkos/kokkos/archive/refs/tags/3.4.01.tar.gz > kokkos-3.4.01.tar.gz && \
tar -xf kokkos-3.4.01.tar.gz && \
curl -L https://github.com/kokkos/kokkos/archive/refs/tags/$KOKKOS_VERSION.tar.gz > kokkos-$KOKKOS_VERSION.tar.gz && \
tar -xf kokkos-$KOKKOS_VERSION.tar.gz && \
mkdir bld && cd bld && \
cmake -GNinja -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC -DKokkos_ENABLE_SERIAL=ON ../kokkos-3.4.01 &&\
cmake -GNinja -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC -DKokkos_ENABLE_SERIAL=ON ../kokkos-$KOKKOS_VERSION &&\
ninja all && \
ninja install

@ -322,7 +322,7 @@ endfunction()
if(VTKm_ENABLE_KOKKOS AND NOT TARGET vtkm_kokkos)
cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
find_package(Kokkos REQUIRED)
find_package(Kokkos 3.7 REQUIRED)
# We must empty this property for every kokkos backend device since it
# contains a generator expresion which breaks some of our users builds.

@ -76,6 +76,8 @@ VTK-m Requires:
Optional dependencies are:
+ Kokkos Device Adapter
+ [Kokkos](https://kokkos.github.io/) 3.7+
+ CUDA Device Adapter
+ [Cuda Toolkit 9.2, >= 10.2](https://developer.nvidia.com/cuda-toolkit)
+ Note CUDA >= 10.2 is required on Windows

@ -0,0 +1,4 @@
# Require Kokkos 3.7
The minimum version of Kokkos supported is now set to Kokkos 3.7. This is
to synchronize with the development of the Kokkos team.

@ -97,13 +97,6 @@ public:
throw vtkm::cont::ErrorBadDevice("Tried to set the number of threads on an invalid device");
}
VTKM_CONT virtual vtkm::cont::internal::RuntimeDeviceConfigReturnCode SetNumaRegions(
const vtkm::Id&) override final
{
throw vtkm::cont::ErrorBadDevice(
"Tried to set the number of numa regions on an invalid device");
}
VTKM_CONT virtual vtkm::cont::internal::RuntimeDeviceConfigReturnCode SetDeviceInstance(
const vtkm::Id&) override final
{
@ -116,13 +109,6 @@ public:
throw vtkm::cont::ErrorBadDevice("Tried to get the number of threads on an invalid device");
}
VTKM_CONT virtual vtkm::cont::internal::RuntimeDeviceConfigReturnCode GetNumaRegions(
vtkm::Id&) const override final
{
throw vtkm::cont::ErrorBadDevice(
"Tried to get the number of numa regions on an invalid device");
}
VTKM_CONT virtual vtkm::cont::internal::RuntimeDeviceConfigReturnCode GetDeviceInstance(
vtkm::Id&) const override final
{

@ -90,11 +90,6 @@ void RuntimeDeviceConfigurationBase::Initialize(
[&](const vtkm::Id& value) { return this->SetThreads(value); },
"SetThreads",
this->GetDevice().GetName());
InitializeOption(
configOptions.VTKmNumaRegions,
[&](const vtkm::Id& value) { return this->SetNumaRegions(value); },
"SetNumaRegions",
this->GetDevice().GetName());
InitializeOption(
configOptions.VTKmDeviceInstance,
[&](const vtkm::Id& value) { return this->SetDeviceInstance(value); },
@ -117,11 +112,6 @@ RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::SetThreads(const v
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;
}
RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::SetNumaRegions(const vtkm::Id&)
{
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;
}
RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::SetDeviceInstance(const vtkm::Id&)
{
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;
@ -132,11 +122,6 @@ RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::GetThreads(vtkm::I
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;
}
RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::GetNumaRegions(vtkm::Id&) const
{
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;
}
RuntimeDeviceConfigReturnCode RuntimeDeviceConfigurationBase::GetDeviceInstance(vtkm::Id&) const
{
return RuntimeDeviceConfigReturnCode::INVALID_FOR_DEVICE;

@ -54,13 +54,11 @@ public:
/// A method should return INVALID_FOR_DEVICE if the overriden device does not
/// support the particular set method.
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetThreads(const vtkm::Id& value);
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetNumaRegions(const vtkm::Id& value);
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetDeviceInstance(const vtkm::Id& value);
/// The following public methods are overriden in each individual device and store the
/// values that were set via the above Set* methods for the given device.
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetThreads(vtkm::Id& value) const;
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetNumaRegions(vtkm::Id& value) const;
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance(vtkm::Id& value) const;
/// The following public methods should be overriden as needed for each individual device

@ -31,13 +31,13 @@ void AppendOptionDescriptors(std::vector<option::Descriptor>& usage,
"vtkm-num-threads",
option::VtkmArg::Required,
" --vtkm-num-threads <dev> \tSets the number of threads to use for the selected device" });
usage.push_back(
{ useOptionIndex ? static_cast<uint32_t>(option::OptionIndex::NUMA_REGIONS) : 1,
0,
"",
"vtkm-numa-regions",
option::VtkmArg::Required,
" --vtkm-numa-regions <dev> \tSets the number of numa regions when using kokkos/OpenMP" });
usage.push_back({ useOptionIndex ? static_cast<uint32_t>(option::OptionIndex::NUMA_REGIONS) : 1,
0,
"",
"vtkm-numa-regions",
option::VtkmArg::Required,
" --vtkm-numa-regions <dev> \tSets the number of numa regions when using "
"kokkos/OpenMP (deprecated, has no effect)" });
usage.push_back(
{ useOptionIndex ? static_cast<uint32_t>(option::OptionIndex::DEVICE_INSTANCE) : 2,
0,
@ -51,7 +51,6 @@ void AppendOptionDescriptors(std::vector<option::Descriptor>& usage,
RuntimeDeviceConfigurationOptions::RuntimeDeviceConfigurationOptions(const bool& useOptionIndex)
: VTKmNumThreads(useOptionIndex ? option::OptionIndex::NUM_THREADS : 0, "VTKM_NUM_THREADS")
, VTKmNumaRegions(useOptionIndex ? option::OptionIndex::NUMA_REGIONS : 1, "VTKM_NUMA_REGIONS")
, VTKmDeviceInstance(useOptionIndex ? option::OptionIndex::DEVICE_INSTANCE : 2,
"VTKM_DEVICE_INSTANCE")
, Initialized(false)
@ -100,7 +99,6 @@ RuntimeDeviceConfigurationOptions::~RuntimeDeviceConfigurationOptions() noexcept
void RuntimeDeviceConfigurationOptions::Initialize(const option::Option* options)
{
this->VTKmNumThreads.Initialize(options);
this->VTKmNumaRegions.Initialize(options);
this->VTKmDeviceInstance.Initialize(options);
this->Initialized = true;
}

@ -48,7 +48,6 @@ public:
VTKM_CONT bool IsInitialized() const;
RuntimeDeviceOption VTKmNumThreads;
RuntimeDeviceOption VTKmNumaRegions;
RuntimeDeviceOption VTKmDeviceInstance;
protected:

@ -726,6 +726,13 @@ private:
template <typename T>
VTKM_CONT static void SortImpl(vtkm::cont::ArrayHandle<T>& values, vtkm::SortLess, std::true_type)
{
// In Kokkos 3.7, we have noticed some errors when sorting with zero-length arrays (which
// should do nothing). There is no check, and the bin size computation gets messed up.
if (values.GetNumberOfValues() <= 1)
{
return;
}
vtkm::cont::Token token;
auto portal = values.PrepareForInPlace(vtkm::cont::DeviceAdapterTagKokkos{}, token);
kokkos::internal::KokkosViewExec<T> view(portal.GetArray(), portal.GetNumberOfValues());

@ -95,22 +95,7 @@ public:
return RuntimeDeviceConfigReturnCode::NOT_APPLIED;
}
this->KokkosArguments.insert(this->KokkosArguments.begin(),
"--kokkos-threads=" + std::to_string(value));
return RuntimeDeviceConfigReturnCode::SUCCESS;
}
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetNumaRegions(
const vtkm::Id& value) override final
{
if (Kokkos::is_initialized())
{
VTKM_LOG_S(vtkm::cont::LogLevel::Warn,
"SetNumaRegions was called but Kokkos was already initailized! Updates will not "
"be applied.");
return RuntimeDeviceConfigReturnCode::NOT_APPLIED;
}
this->KokkosArguments.insert(this->KokkosArguments.begin(),
"--kokkos-numa=" + std::to_string(value));
"--kokkos-num-threads=" + std::to_string(value));
return RuntimeDeviceConfigReturnCode::SUCCESS;
}
@ -131,13 +116,7 @@ public:
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetThreads(vtkm::Id& value) const override final
{
return GetArgFromList(this->KokkosArguments, "--kokkos-threads", value);
}
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetNumaRegions(
vtkm::Id& value) const override final
{
return GetArgFromList(this->KokkosArguments, "--kokkos-numa", value);
return GetArgFromList(this->KokkosArguments, "--kokkos-num-threads", value);
}
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance(
@ -149,7 +128,7 @@ public:
protected:
/// Store a copy of the current arguments when initializing the Kokkos subsystem later
/// Appends a copy of the argv values in the KokkosArguments vector: this assumes the
/// argv values contain kokkos command line arguments (like --kokkos-threads, etc)
/// argv values contain kokkos command line arguments (like --kokkos-num-threads, etc)
VTKM_CONT virtual void ParseExtraArguments(int& argc, char* argv[]) override final
{
if (argc > 0 && argv)

@ -26,22 +26,8 @@ TestingRuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagKokkos>::TestRunti
{
int argc;
char** argv;
vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-numa=4");
vtkm::cont::testing::Testing::SetEnv("KOKKOS_DEVICE_ID", "0");
vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-print-configuration");
auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions();
bool threw = false;
try
{
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
DeviceAdapterTagKokkos(), deviceOptions, argc, argv);
}
catch (const std::runtime_error& e)
{
threw = true;
}
VTKM_TEST_ASSERT(threw,
"GetRuntimeConfiguration should have thrown, env KOKKOS_DEVICE_ID didn't match");
VTKM_TEST_ASSERT(!Kokkos::is_initialized(), "Kokkos should not be initialized at this point");
deviceOptions.VTKmDeviceInstance.SetOption(0);
internal::RuntimeDeviceConfigurationBase& config =
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
@ -54,28 +40,21 @@ TestingRuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagKokkos>::TestRunti
"Failed to get set threads");
VTKM_TEST_ASSERT(testValue == 8,
"Set threads does not match expected value: 8 != " + std::to_string(testValue));
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
"Failed to get set numa regions");
VTKM_TEST_ASSERT(testValue == 4,
"Set numa regions does not match expected value: 4 != " +
std::to_string(testValue));
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
"Failed to get set device instance");
VTKM_TEST_ASSERT(testValue == 0,
"Set device instance does not match expected value: 0 != " +
std::to_string(testValue));
// Ensure that with kokkos we can't re-initialize or set values after the first initialize
// Should pop up a few warnings in the test logs
std::cout
<< "Ensure that with kokkos we can't re-initialize or set values after the first initialize"
<< std::endl;
std::cout << "This should pop up a few warnings in the test logs" << std::endl;
deviceOptions.VTKmNumThreads.SetOption(16);
deviceOptions.VTKmNumaRegions.SetOption(2);
deviceOptions.VTKmDeviceInstance.SetOption(5);
config.Initialize(deviceOptions);
VTKM_TEST_ASSERT(config.SetThreads(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
"Shouldn't be able to set threads after kokkos is initalized");
VTKM_TEST_ASSERT(config.SetNumaRegions(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
"Shouldn't be able to set numa regions after kokkos is initalized");
VTKM_TEST_ASSERT(config.SetDeviceInstance(1) ==
internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
"Shouldn't be able to set device instnace after kokkos is initalized");
@ -85,20 +64,12 @@ TestingRuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagKokkos>::TestRunti
"Failed to get set threads");
VTKM_TEST_ASSERT(testValue == 8,
"Set threads does not match expected value: 8 != " + std::to_string(testValue));
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
"Failed to get set numa regions");
VTKM_TEST_ASSERT(testValue == 4,
"Set numa regions does not match expected value: 4 != " +
std::to_string(testValue));
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
"Failed to get set device instance");
VTKM_TEST_ASSERT(testValue == 0,
"Set device instance does not match expected value: 0 != " +
std::to_string(testValue));
vtkm::cont::testing::Testing::UnsetEnv("KOKKOS_DEVICE_ID");
}
} // namespace vtkm::cont::testing

@ -35,7 +35,6 @@ struct TestingRuntimeDeviceConfiguration
{
internal::RuntimeDeviceConfigurationOptions runtimeDeviceOptions{};
runtimeDeviceOptions.VTKmNumThreads.SetOption(8);
runtimeDeviceOptions.VTKmNumaRegions.SetOption(0);
runtimeDeviceOptions.VTKmDeviceInstance.SetOption(2);
runtimeDeviceOptions.Initialize(nullptr);
VTKM_TEST_ASSERT(runtimeDeviceOptions.IsInitialized(),

@ -166,6 +166,7 @@ void DoTest()
constexpr vtkm::Id STRIDE = 7;
vtkm::cont::ArrayHandleBasic<vtkm::Vec3f> originalArray;
originalArray.Allocate(ARRAY_SIZE * STRIDE);
SetPortal(originalArray.WritePortal());
for (vtkm::Id offset = 0; offset < STRIDE; ++offset)
{
vtkm::cont::ArrayHandleStride<vtkm::Vec3f> strideArray(

@ -146,16 +146,8 @@ void InitializeRuntimeDeviceConfigurationWithArgs()
{
int argc;
char** argv;
vtkm::cont::testing::Testing::MakeArgsAddProgramName(argc,
argv,
"--vtkm-device",
"Any",
"--vtkm-num-threads",
"100",
"--vtkm-numa-regions",
"4",
"--vtkm-device-instance",
"2");
vtkm::cont::testing::Testing::MakeArgsAddProgramName(
argc, argv, "--vtkm-device", "Any", "--vtkm-num-threads", "100", "--vtkm-device-instance", "2");
vtkm::cont::Initialize(argc, argv);
CheckArgs(argc, argv);
}

@ -187,11 +187,9 @@ void TestConfigOptionValues(const internal::RuntimeDeviceConfigurationOptions& c
VTKM_TEST_ASSERT(configOptions.IsInitialized(), "runtime config options should be initialized");
VTKM_TEST_ASSERT(configOptions.VTKmNumThreads.IsSet(), "num threads should be set");
VTKM_TEST_ASSERT(configOptions.VTKmNumaRegions.IsSet(), "numa regions should be set");
VTKM_TEST_ASSERT(configOptions.VTKmDeviceInstance.IsSet(), "device instance should be set");
VTKM_TEST_ASSERT(configOptions.VTKmNumThreads.GetValue() == 100, "num threads should == 100");
VTKM_TEST_ASSERT(configOptions.VTKmNumaRegions.GetValue() == 2, "numa regions should == 2");
VTKM_TEST_ASSERT(configOptions.VTKmDeviceInstance.GetValue() == 1, "device instance should == 1");
}
@ -211,14 +209,8 @@ void TestRuntimeDeviceConfigurationOptions()
int argc;
char** argv;
vtkm::cont::testing::Testing::MakeArgs(argc,
argv,
"--vtkm-num-threads",
"100",
"--vtkm-numa-regions",
"2",
"--vtkm-device-instance",
"1");
vtkm::cont::testing::Testing::MakeArgs(
argc, argv, "--vtkm-num-threads", "100", "--vtkm-device-instance", "1");
auto options = GetOptions(argc, argv, usage);
VTKM_TEST_ASSERT(!configOptions.IsInitialized(),
@ -230,14 +222,8 @@ void TestRuntimeDeviceConfigurationOptions()
{
int argc;
char** argv;
vtkm::cont::testing::Testing::MakeArgs(argc,
argv,
"--vtkm-num-threads",
"100",
"--vtkm-numa-regions",
"2",
"--vtkm-device-instance",
"1");
vtkm::cont::testing::Testing::MakeArgs(
argc, argv, "--vtkm-num-threads", "100", "--vtkm-device-instance", "1");
internal::RuntimeDeviceConfigurationOptions configOptions(argc, argv);
TestConfigOptionValues(configOptions);
}