From adac415f1579c4361a698c31c038543e837ff7d8 Mon Sep 17 00:00:00 2001 From: Nickolas Davis Date: Thu, 26 Aug 2021 13:27:03 -0600 Subject: [PATCH] implement cuda runtime device configuraton --- vtkm/cont/RuntimeDeviceInformation.cxx | 2 +- vtkm/cont/cuda/ChooseCudaDevice.h | 48 ++++--- vtkm/cont/cuda/internal/CudaAllocator.cu | 23 +++- .../internal/RuntimeDeviceConfigurationCuda.h | 68 +++++++++- vtkm/cont/cuda/testing/CMakeLists.txt | 1 + .../UnitTestCudaRuntimeDeviceConfiguration.cu | 64 +++++++++ .../internal/RuntimeDeviceConfiguration.h | 5 +- .../RuntimeDeviceConfigurationKokkos.h | 1 + ...itTestKokkosRuntimeDeviceConfiguration.cxx | 95 +++++++++++++- .../TestingRuntimeDeviceConfiguration.h | 123 +----------------- 10 files changed, 275 insertions(+), 155 deletions(-) create mode 100644 vtkm/cont/cuda/testing/UnitTestCudaRuntimeDeviceConfiguration.cu diff --git a/vtkm/cont/RuntimeDeviceInformation.cxx b/vtkm/cont/RuntimeDeviceInformation.cxx index bd97adae0..40ddaa8c4 100644 --- a/vtkm/cont/RuntimeDeviceInformation.cxx +++ b/vtkm/cont/RuntimeDeviceInformation.cxx @@ -391,7 +391,7 @@ private: } RuntimeDeviceConfigurations( - const vtkm::cont::internal::RuntimeDeviceConfigurationOptions configOptions, + const vtkm::cont::internal::RuntimeDeviceConfigurationOptions& configOptions, int& argc, char* argv[]) { diff --git a/vtkm/cont/cuda/ChooseCudaDevice.h b/vtkm/cont/cuda/ChooseCudaDevice.h index 7c9ca6211..ae3a215db 100644 --- a/vtkm/cont/cuda/ChooseCudaDevice.h +++ b/vtkm/cont/cuda/ChooseCudaDevice.h @@ -12,9 +12,13 @@ #include +#include #include +#include +#include #include +#include #include VTKM_THIRDPARTY_PRE_INCLUDE @@ -91,35 +95,43 @@ private: ///A result of zero means no cuda device has been found static int FindFastestDeviceId() { - //get the number of devices and store information - int numberOfDevices = 0; - VTKM_CUDA_CALL(cudaGetDeviceCount(&numberOfDevices)); + auto cudaDeviceConfig = dynamic_cast< + vtkm::cont::internal::RuntimeDeviceConfiguration&>( + vtkm::cont::RuntimeDeviceInformation{}.GetRuntimeConfiguration( + vtkm::cont::DeviceAdapterTagCuda())); + vtkm::Id numDevices; + cudaDeviceConfig.GetMaxDevices(numDevices); - std::vector devices; - for (int i = 0; i < numberOfDevices; ++i) + // multiset stores elements in sorted order (allows duplicate values) + std::multiset devices; + std::vector cudaProp; + cudaDeviceConfig.GetCudaDeviceProp(cudaProp); + for (int i = 0; i < numDevices; ++i) { - cudaDeviceProp properties; - VTKM_CUDA_CALL(cudaGetDeviceProperties(&properties, i)); - if (properties.computeMode != cudaComputeModeProhibited) + if (cudaProp[i].computeMode != cudaComputeModeProhibited) { - //only add devices that have compute mode allowed - devices.push_back(compute_info(properties, i)); + devices.emplace(cudaProp[i], i); } } - //sort from fastest to slowest - std::sort(devices.begin(), devices.end()); + return devices.size() > 0 ? devices.begin()->GetIndex() : 0; +} - int device = 0; - if (devices.size() > 0) - { - device = devices.front().GetIndex(); - } - return device; +/// Sets the current cuda device to the value returned by FindFastestDeviceId +static void SetFastestDeviceId() +{ + auto deviceId = FindFastestDeviceId(); + vtkm::cont::RuntimeDeviceInformation{} + .GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda()) + .SetDeviceInstance(deviceId); } //choose a cuda compute device. This can't be used if you are setting //up open gl interop +VTKM_DEPRECATED(1.7, + "Use " + "RuntimeInformation{}.GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda)." + "SetDeviceInstance(id) instead") static void SetCudaDevice(int id) { cudaError_t cError = cudaSetDevice(id); diff --git a/vtkm/cont/cuda/internal/CudaAllocator.cu b/vtkm/cont/cuda/internal/CudaAllocator.cu index 9d443bca0..aec7cbff3 100644 --- a/vtkm/cont/cuda/internal/CudaAllocator.cu +++ b/vtkm/cont/cuda/internal/CudaAllocator.cu @@ -11,8 +11,11 @@ #include #include #include +#include #include #include +#include +#include #define NO_VTKM_MANAGED_MEMORY "NO_VTKM_MANAGED_MEMORY" #include @@ -266,8 +269,10 @@ void CudaAllocator::PrepareForInPlace(const void* ptr, std::size_t numBytes) if (IsManagedPointer(ptr) && numBytes >= Threshold) { #if CUDART_VERSION >= 8000 - int dev; - VTKM_CUDA_CALL(cudaGetDevice(&dev)); + vtkm::Id dev; + vtkm::cont::RuntimeDeviceInformation() + .GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda()) + .GetDeviceInstance(dev); // VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseSetPreferredLocation, dev)); // VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseUnsetReadMostly, dev)); VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseSetAccessedBy, dev)); @@ -280,8 +285,12 @@ void CudaAllocator::Initialize() { #if CUDART_VERSION >= 8000 std::call_once(IsInitialized, []() { - int numDevices; - VTKM_CUDA_CALL(cudaGetDeviceCount(&numDevices)); + auto cudaDeviceConfig = dynamic_cast< + vtkm::cont::internal::RuntimeDeviceConfiguration&>( + vtkm::cont::RuntimeDeviceInformation{}.GetRuntimeConfiguration( + vtkm::cont::DeviceAdapterTagCuda())); + vtkm::Id numDevices; + cudaDeviceConfig.GetMaxDevices(numDevices); if (numDevices == 0) { @@ -290,13 +299,13 @@ void CudaAllocator::Initialize() // Check all devices, use the feature set supported by all bool managedMemorySupported = true; - cudaDeviceProp prop; + std::vector cudaProp; + cudaDeviceConfig.GetCudaDeviceProp(cudaProp); for (int i = 0; i < numDevices && managedMemorySupported; ++i) { - VTKM_CUDA_CALL(cudaGetDeviceProperties(&prop, i)); // We check for concurrentManagedAccess, as devices with only the // managedAccess property have extra synchronization requirements. - managedMemorySupported = managedMemorySupported && prop.concurrentManagedAccess; + managedMemorySupported = managedMemorySupported && cudaProp[i].concurrentManagedAccess; } HardwareSupportsManagedMemory = managedMemorySupported; diff --git a/vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h b/vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h index a4369e915..513b63d24 100644 --- a/vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h +++ b/vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h @@ -13,6 +13,15 @@ #include #include +#include +#include + +VTKM_THIRDPARTY_PRE_INCLUDE +#include +VTKM_THIRDPARTY_POST_INCLUDE + +#include + namespace vtkm { namespace cont @@ -24,27 +33,76 @@ template <> class RuntimeDeviceConfiguration : public vtkm::cont::internal::RuntimeDeviceConfigurationBase { +public: + RuntimeDeviceConfiguration() + { + int tmp; + VTKM_CUDA_CALL(cudaGetDeviceCount(&tmp)); + this->CudaDeviceCount = tmp; + this->CudaProp.resize(this->CudaDeviceCount); + for (int i = 0; i < this->CudaDeviceCount; ++i) + { + VTKM_CUDA_CALL(cudaGetDeviceProperties(&this->CudaProp[i], i)); + } + } + VTKM_CONT vtkm::cont::DeviceAdapterId GetDevice() const override final { return vtkm::cont::DeviceAdapterTagCuda{}; } - VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetDeviceInstance(const vtkm::Id&) override final + VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetDeviceInstance( + const vtkm::Id& value) override final { - // TODO: set the cuda device instance + if (value >= this->CudaDeviceCount) + { + VTKM_LOG_S( + vtkm::cont::LogLevel::Error, + "Failed to set CudaDeviceInstance, supplied id exceeds the number of available devices: " + << value << " >= " << this->CudaDeviceCount); + return RuntimeDeviceConfigReturnCode::INVALID_VALUE; + } + try + { + VTKM_CUDA_CALL(cudaSetDevice(value)); + return RuntimeDeviceConfigReturnCode::SUCCESS; + } + catch (const vtkm::cont::cuda::ErrorCuda& err) + { + VTKM_LOG_S(vtkm::cont::LogLevel::Error, + "Failed to set CudaDeviceInstance: " << err.GetMessage()); + return RuntimeDeviceConfigReturnCode::INTERNAL_ERROR; + } + } + + VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance( + vtkm::Id& value) const override final + { + int tmp; + VTKM_CUDA_CALL(cudaGetDevice(&tmp)); + value = tmp; return RuntimeDeviceConfigReturnCode::SUCCESS; } - VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance(vtkm::Id&) const override final + VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetMaxDevices( + vtkm::Id& value) const override final { - // TODO: Get the cuda device instance (also maybe a list of available devices?) + value = this->CudaDeviceCount; return RuntimeDeviceConfigReturnCode::SUCCESS; } - VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetMaxDevices(vtkm::Id&) const override final + /// A function only available for use by the Cuda instance of this class + /// Used to grab the CudaDeviceProp structs for all available devices + VTKM_CONT RuntimeDeviceConfigReturnCode + GetCudaDeviceProp(std::vector& value) const { + value = CudaProp; return RuntimeDeviceConfigReturnCode::SUCCESS; } + +private: + std::vector CudaProp; + vtkm::Id CudaDeviceCount; }; } // namespace vtkm::cont::internal } // namespace vtkm::cont diff --git a/vtkm/cont/cuda/testing/CMakeLists.txt b/vtkm/cont/cuda/testing/CMakeLists.txt index f5abbcf22..eef7d335b 100644 --- a/vtkm/cont/cuda/testing/CMakeLists.txt +++ b/vtkm/cont/cuda/testing/CMakeLists.txt @@ -26,6 +26,7 @@ set(unit_tests UnitTestCudaMathEdgeCases.cu UnitTestCudaShareUserProvidedManagedMemory.cu UnitTestCudaPointLocatorSparseGrid.cu + UnitTestCudaRuntimeDeviceConfiguration.cu ) if (NOT VTKm_NO_DEPRECATED_VIRTUAL) diff --git a/vtkm/cont/cuda/testing/UnitTestCudaRuntimeDeviceConfiguration.cu b/vtkm/cont/cuda/testing/UnitTestCudaRuntimeDeviceConfiguration.cu new file mode 100644 index 000000000..1d6070cd1 --- /dev/null +++ b/vtkm/cont/cuda/testing/UnitTestCudaRuntimeDeviceConfiguration.cu @@ -0,0 +1,64 @@ +//============================================================================ +// Copyright (c) Kitware, Inc. +// All rights reserved. +// See LICENSE.txt for details. +// +// This software is distributed WITHOUT ANY WARRANTY; without even +// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. See the above copyright notice for more information. +//============================================================================ +#include +#include +#include + +namespace internal = vtkm::cont::internal; + +namespace vtkm +{ +namespace cont +{ +namespace testing +{ + +template <> +VTKM_CONT void +TestingRuntimeDeviceConfiguration::TestRuntimeConfig() +{ + auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions(); + int numDevices = 0; + VTKM_CUDA_CALL(cudaGetDeviceCount(&numDevices)); + vtkm::Id selectedDevice = numDevices > 0 ? numDevices - 1 : 0; + deviceOptions.VTKmDeviceInstance.SetOption(selectedDevice); + auto& config = + RuntimeDeviceInformation{}.GetRuntimeConfiguration(DeviceAdapterTagCuda(), deviceOptions); + vtkm::Id setDevice; + VTKM_TEST_ASSERT(config.GetDeviceInstance(setDevice) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get device instance"); + VTKM_TEST_ASSERT(setDevice == selectedDevice, + "RTC's setDevice != selectedDevice cuda direct! " + std::to_string(setDevice) + + " != " + std::to_string(selectedDevice)); + vtkm::Id maxDevices; + VTKM_TEST_ASSERT(config.GetMaxDevices(maxDevices) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get max devices"); + VTKM_TEST_ASSERT(maxDevices == numDevices, + "RTC's maxDevices != numDevices cuda direct! " + std::to_string(maxDevices) + + " != " + std::to_string(numDevices)); + std::vector cudaProps; + dynamic_cast&>(config) + .GetCudaDeviceProp(cudaProps); + VTKM_TEST_ASSERT(maxDevices == static_cast(cudaProps.size()), + "CudaProp's size != maxDevices! " + std::to_string(cudaProps.size()) + + " != " + std::to_string(maxDevices)); +} + +} // namespace vtkm::cont::testing +} // namespace vtkm::cont +} // namespace vtkm + +int UnitTestCudaRuntimeDeviceConfiguration(int argc, char* argv[]) +{ + return vtkm::cont::testing::TestingRuntimeDeviceConfiguration< + vtkm::cont::DeviceAdapterTagCuda>::Run(argc, argv); +} diff --git a/vtkm/cont/internal/RuntimeDeviceConfiguration.h b/vtkm/cont/internal/RuntimeDeviceConfiguration.h index f71623d3d..cc264173c 100644 --- a/vtkm/cont/internal/RuntimeDeviceConfiguration.h +++ b/vtkm/cont/internal/RuntimeDeviceConfiguration.h @@ -24,13 +24,14 @@ namespace cont namespace internal { -enum class RuntimeDeviceConfigReturnCode +enum class RuntimeDeviceConfigReturnCode : vtkm::Id { SUCCESS, OUT_OF_BOUNDS, INVALID_FOR_DEVICE, INVALID_VALUE, - NOT_APPLIED + NOT_APPLIED, + INTERNAL_ERROR }; class VTKM_CONT_EXPORT RuntimeDeviceConfigurationBase diff --git a/vtkm/cont/kokkos/internal/RuntimeDeviceConfigurationKokkos.h b/vtkm/cont/kokkos/internal/RuntimeDeviceConfigurationKokkos.h index d143b47b2..de073480f 100644 --- a/vtkm/cont/kokkos/internal/RuntimeDeviceConfigurationKokkos.h +++ b/vtkm/cont/kokkos/internal/RuntimeDeviceConfigurationKokkos.h @@ -11,6 +11,7 @@ #define vtk_m_cont_kokkos_internal_RuntimeDeviceConfigurationKokkos_h #include +#include #include #include diff --git a/vtkm/cont/kokkos/testing/UnitTestKokkosRuntimeDeviceConfiguration.cxx b/vtkm/cont/kokkos/testing/UnitTestKokkosRuntimeDeviceConfiguration.cxx index cf5d716e0..d201dba9c 100644 --- a/vtkm/cont/kokkos/testing/UnitTestKokkosRuntimeDeviceConfiguration.cxx +++ b/vtkm/cont/kokkos/testing/UnitTestKokkosRuntimeDeviceConfiguration.cxx @@ -11,9 +11,102 @@ #include #include +namespace internal = vtkm::cont::internal; + +namespace vtkm +{ +namespace cont +{ +namespace testing +{ + +template <> +VTKM_CONT void +TestingRuntimeDeviceConfiguration::TestRuntimeConfig() +{ + int argc; + char** argv; + vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-numa=4"); + vtkm::cont::testing::Testing::SetEnv("KOKKOS_DEVICE_ID", "0"); + auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions(); + bool threw = false; + try + { + RuntimeDeviceInformation{}.GetRuntimeConfiguration( + DeviceAdapterTagKokkos(), deviceOptions, argc, argv); + } + catch (const std::runtime_error& e) + { + threw = true; + } + VTKM_TEST_ASSERT(threw, + "GetRuntimeConfiguration should have thrown, env KOKKOS_DEVICE_ID didn't match"); + VTKM_TEST_ASSERT(!Kokkos::is_initialized(), "Kokkos should not be initialized at this point"); + deviceOptions.VTKmDeviceInstance.SetOption(0); + internal::RuntimeDeviceConfigurationBase& config = + RuntimeDeviceInformation{}.GetRuntimeConfiguration( + DeviceAdapterTagKokkos(), deviceOptions, argc, argv); + VTKM_TEST_ASSERT(Kokkos::is_initialized(), "Kokkos should be initialized at this point"); + + // Test that args are set and the right arg priority is applied + vtkm::Id testValue; + VTKM_TEST_ASSERT(config.GetThreads(testValue) == internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set threads"); + VTKM_TEST_ASSERT(testValue == 8, + "Set threads does not match expected value: 8 != " + std::to_string(testValue)); + VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set numa regions"); + VTKM_TEST_ASSERT(testValue == 4, + "Set numa regions does not match expected value: 4 != " + + std::to_string(testValue)); + VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set device instance"); + VTKM_TEST_ASSERT(testValue == 0, + "Set device instance does not match expected value: 0 != " + + std::to_string(testValue)); + // Ensure that with kokkos we can't re-initialize or set values after the first initialize + // Should pop up a few warnings in the test logs + deviceOptions.VTKmNumThreads.SetOption(16); + deviceOptions.VTKmNumaRegions.SetOption(2); + deviceOptions.VTKmDeviceInstance.SetOption(5); + config.Initialize(deviceOptions); + VTKM_TEST_ASSERT(config.SetThreads(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, + "Shouldn't be able to set threads after kokkos is initalized"); + VTKM_TEST_ASSERT(config.SetNumaRegions(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, + "Shouldn't be able to set numa regions after kokkos is initalized"); + VTKM_TEST_ASSERT(config.SetDeviceInstance(1) == + internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, + "Shouldn't be able to set device instnace after kokkos is initalized"); + + // make sure all the values are the same + VTKM_TEST_ASSERT(config.GetThreads(testValue) == internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set threads"); + VTKM_TEST_ASSERT(testValue == 8, + "Set threads does not match expected value: 8 != " + std::to_string(testValue)); + VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set numa regions"); + VTKM_TEST_ASSERT(testValue == 4, + "Set numa regions does not match expected value: 4 != " + + std::to_string(testValue)); + VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) == + internal::RuntimeDeviceConfigReturnCode::SUCCESS, + "Failed to get set device instance"); + VTKM_TEST_ASSERT(testValue == 0, + "Set device instance does not match expected value: 0 != " + + std::to_string(testValue)); + + vtkm::cont::testing::Testing::UnsetEnv("KOKKOS_DEVICE_ID"); +} + +} // namespace vtkm::cont::testing +} // namespace vtkm::cont +} // namespace vtkm + int UnitTestKokkosRuntimeDeviceConfiguration(int argc, char* argv[]) { - vtkm::cont::GetRuntimeDeviceTracker().ForceDevice(vtkm::cont::DeviceAdapterTagKokkos{}); return vtkm::cont::testing::TestingRuntimeDeviceConfiguration< vtkm::cont::DeviceAdapterTagKokkos>::Run(argc, argv); } diff --git a/vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h b/vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h index 4a050c750..6d5677289 100644 --- a/vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h +++ b/vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h @@ -43,137 +43,18 @@ struct TestingRuntimeDeviceConfiguration return runtimeDeviceOptions; } - VTKM_CONT - static void TestSerial() {} - - VTKM_CONT - static void TestTBB() {} - - VTKM_CONT - static void TestOpenMP() {} - - VTKM_CONT - static void TestCuda() {} - - VTKM_CONT - static void TestKokkos() - { - int argc; - char** argv; - vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-numa=4"); - vtkm::cont::testing::Testing::SetEnv("KOKKOS_DEVICE_ID", "0"); - auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions(); - bool threw = false; - try - { - RuntimeDeviceInformation{}.GetRuntimeConfiguration( - DeviceAdapterTag(), deviceOptions, argc, argv); - } - catch (const std::runtime_error& e) - { - threw = true; - } - VTKM_TEST_ASSERT( - threw, "GetRuntimeConfiguration should have thrown, env KOKKOS_DEVICE_ID didn't match"); - VTKM_TEST_ASSERT(!Kokkos::is_initialized(), "Kokkos should not be initialized at this point"); - deviceOptions.VTKmDeviceInstance.SetOption(0); - internal::RuntimeDeviceConfigurationBase& config = - RuntimeDeviceInformation{}.GetRuntimeConfiguration( - DeviceAdapterTag(), deviceOptions, argc, argv); - VTKM_TEST_ASSERT(Kokkos::is_initialized(), "Kokkos should be initialized at this point"); - - // Test that args are set and the right arg priority is applied - vtkm::Id testValue; - VTKM_TEST_ASSERT(config.GetThreads(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set threads"); - VTKM_TEST_ASSERT(testValue == 8, - "Set threads does not match expected value: 8 != " + - std::to_string(testValue)); - VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set numa regions"); - VTKM_TEST_ASSERT(testValue == 4, - "Set numa regions does not match expected value: 4 != " + - std::to_string(testValue)); - VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set device instance"); - VTKM_TEST_ASSERT(testValue == 0, - "Set device instance does not match expected value: 0 != " + - std::to_string(testValue)); - - // Ensure that with kokkos we can't re-initialize or set values after the first initialize - // Should pop up a few warnings in the test logs - deviceOptions.VTKmNumThreads.SetOption(16); - deviceOptions.VTKmNumaRegions.SetOption(2); - deviceOptions.VTKmDeviceInstance.SetOption(5); - config.Initialize(deviceOptions); - VTKM_TEST_ASSERT(config.SetThreads(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, - "Shouldn't be able to set threads after kokkos is initalized"); - VTKM_TEST_ASSERT(config.SetNumaRegions(1) == - internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, - "Shouldn't be able to set numa regions after kokkos is initalized"); - VTKM_TEST_ASSERT(config.SetDeviceInstance(1) == - internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED, - "Shouldn't be able to set device instnace after kokkos is initalized"); - - // make sure all the values are the same - VTKM_TEST_ASSERT(config.GetThreads(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set threads"); - VTKM_TEST_ASSERT(testValue == 8, - "Set threads does not match expected value: 8 != " + - std::to_string(testValue)); - VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set numa regions"); - VTKM_TEST_ASSERT(testValue == 4, - "Set numa regions does not match expected value: 4 != " + - std::to_string(testValue)); - VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) == - internal::RuntimeDeviceConfigReturnCode::SUCCESS, - "Failed to get set device instance"); - VTKM_TEST_ASSERT(testValue == 0, - "Set device instance does not match expected value: 0 != " + - std::to_string(testValue)); - - vtkm::cont::testing::Testing::UnsetEnv("KOKKOS_DEVICE_ID"); - } + VTKM_CONT static void TestRuntimeConfig(){}; struct TestRunner { VTKM_CONT - void operator()() const - { - switch (DeviceAdapterTag{}.GetValue()) - { - case vtkm::cont::DeviceAdapterTagSerial{}.GetValue(): - TestingRuntimeDeviceConfiguration::TestSerial(); - break; - case vtkm::cont::DeviceAdapterTagTBB{}.GetValue(): - TestingRuntimeDeviceConfiguration::TestTBB(); - break; - case vtkm::cont::DeviceAdapterTagOpenMP{}.GetValue(): - TestingRuntimeDeviceConfiguration::TestOpenMP(); - break; - case vtkm::cont::DeviceAdapterTagCuda{}.GetValue(): - TestingRuntimeDeviceConfiguration::TestCuda(); - break; - case vtkm::cont::DeviceAdapterTagKokkos{}.GetValue(): - TestingRuntimeDeviceConfiguration::TestKokkos(); - break; - default: - break; - } - } + void operator()() const { TestRuntimeConfig(); } }; static VTKM_CONT int Run(int, char*[]) { // For the Kokkos version of this test we don't not want Initialize to be called // so we directly execute the testing functor instead of calling Run - vtkm::cont::GetRuntimeDeviceTracker().ForceDevice(DeviceAdapterTag()); return vtkm::cont::testing::Testing::ExecuteFunction(TestRunner{}); } };