implement cuda runtime device configuraton
This commit is contained in:
parent
9d79bdaf0b
commit
adac415f15
@ -391,7 +391,7 @@ private:
|
||||
}
|
||||
|
||||
RuntimeDeviceConfigurations(
|
||||
const vtkm::cont::internal::RuntimeDeviceConfigurationOptions configOptions,
|
||||
const vtkm::cont::internal::RuntimeDeviceConfigurationOptions& configOptions,
|
||||
int& argc,
|
||||
char* argv[])
|
||||
{
|
||||
|
@ -12,9 +12,13 @@
|
||||
|
||||
#include <vtkm/cont/ErrorExecution.h>
|
||||
|
||||
#include <vtkm/cont/RuntimeDeviceInformation.h>
|
||||
#include <vtkm/cont/cuda/ErrorCuda.h>
|
||||
#include <vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h>
|
||||
#include <vtkm/cont/internal/DeviceAdapterTagCuda.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
VTKM_THIRDPARTY_PRE_INCLUDE
|
||||
@ -91,35 +95,43 @@ private:
|
||||
///A result of zero means no cuda device has been found
|
||||
static int FindFastestDeviceId()
|
||||
{
|
||||
//get the number of devices and store information
|
||||
int numberOfDevices = 0;
|
||||
VTKM_CUDA_CALL(cudaGetDeviceCount(&numberOfDevices));
|
||||
auto cudaDeviceConfig = dynamic_cast<
|
||||
vtkm::cont::internal::RuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>&>(
|
||||
vtkm::cont::RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
vtkm::cont::DeviceAdapterTagCuda()));
|
||||
vtkm::Id numDevices;
|
||||
cudaDeviceConfig.GetMaxDevices(numDevices);
|
||||
|
||||
std::vector<compute_info> devices;
|
||||
for (int i = 0; i < numberOfDevices; ++i)
|
||||
// multiset stores elements in sorted order (allows duplicate values)
|
||||
std::multiset<compute_info> devices;
|
||||
std::vector<cudaDeviceProp> cudaProp;
|
||||
cudaDeviceConfig.GetCudaDeviceProp(cudaProp);
|
||||
for (int i = 0; i < numDevices; ++i)
|
||||
{
|
||||
cudaDeviceProp properties;
|
||||
VTKM_CUDA_CALL(cudaGetDeviceProperties(&properties, i));
|
||||
if (properties.computeMode != cudaComputeModeProhibited)
|
||||
if (cudaProp[i].computeMode != cudaComputeModeProhibited)
|
||||
{
|
||||
//only add devices that have compute mode allowed
|
||||
devices.push_back(compute_info(properties, i));
|
||||
devices.emplace(cudaProp[i], i);
|
||||
}
|
||||
}
|
||||
|
||||
//sort from fastest to slowest
|
||||
std::sort(devices.begin(), devices.end());
|
||||
|
||||
int device = 0;
|
||||
if (devices.size() > 0)
|
||||
{
|
||||
device = devices.front().GetIndex();
|
||||
return devices.size() > 0 ? devices.begin()->GetIndex() : 0;
|
||||
}
|
||||
return device;
|
||||
|
||||
/// Sets the current cuda device to the value returned by FindFastestDeviceId
|
||||
static void SetFastestDeviceId()
|
||||
{
|
||||
auto deviceId = FindFastestDeviceId();
|
||||
vtkm::cont::RuntimeDeviceInformation{}
|
||||
.GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda())
|
||||
.SetDeviceInstance(deviceId);
|
||||
}
|
||||
|
||||
//choose a cuda compute device. This can't be used if you are setting
|
||||
//up open gl interop
|
||||
VTKM_DEPRECATED(1.7,
|
||||
"Use "
|
||||
"RuntimeInformation{}.GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda)."
|
||||
"SetDeviceInstance(id) instead")
|
||||
static void SetCudaDevice(int id)
|
||||
{
|
||||
cudaError_t cError = cudaSetDevice(id);
|
||||
|
@ -11,8 +11,11 @@
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <vtkm/cont/Logging.h>
|
||||
#include <vtkm/cont/RuntimeDeviceInformation.h>
|
||||
#include <vtkm/cont/cuda/ErrorCuda.h>
|
||||
#include <vtkm/cont/cuda/internal/CudaAllocator.h>
|
||||
#include <vtkm/cont/cuda/internal/DeviceAdapterTagCuda.h>
|
||||
#include <vtkm/cont/cuda/internal/RuntimeDeviceConfigurationCuda.h>
|
||||
#define NO_VTKM_MANAGED_MEMORY "NO_VTKM_MANAGED_MEMORY"
|
||||
|
||||
#include <mutex>
|
||||
@ -266,8 +269,10 @@ void CudaAllocator::PrepareForInPlace(const void* ptr, std::size_t numBytes)
|
||||
if (IsManagedPointer(ptr) && numBytes >= Threshold)
|
||||
{
|
||||
#if CUDART_VERSION >= 8000
|
||||
int dev;
|
||||
VTKM_CUDA_CALL(cudaGetDevice(&dev));
|
||||
vtkm::Id dev;
|
||||
vtkm::cont::RuntimeDeviceInformation()
|
||||
.GetRuntimeConfiguration(vtkm::cont::DeviceAdapterTagCuda())
|
||||
.GetDeviceInstance(dev);
|
||||
// VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseSetPreferredLocation, dev));
|
||||
// VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseUnsetReadMostly, dev));
|
||||
VTKM_CUDA_CALL(cudaMemAdvise(ptr, numBytes, cudaMemAdviseSetAccessedBy, dev));
|
||||
@ -280,8 +285,12 @@ void CudaAllocator::Initialize()
|
||||
{
|
||||
#if CUDART_VERSION >= 8000
|
||||
std::call_once(IsInitialized, []() {
|
||||
int numDevices;
|
||||
VTKM_CUDA_CALL(cudaGetDeviceCount(&numDevices));
|
||||
auto cudaDeviceConfig = dynamic_cast<
|
||||
vtkm::cont::internal::RuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>&>(
|
||||
vtkm::cont::RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
vtkm::cont::DeviceAdapterTagCuda()));
|
||||
vtkm::Id numDevices;
|
||||
cudaDeviceConfig.GetMaxDevices(numDevices);
|
||||
|
||||
if (numDevices == 0)
|
||||
{
|
||||
@ -290,13 +299,13 @@ void CudaAllocator::Initialize()
|
||||
|
||||
// Check all devices, use the feature set supported by all
|
||||
bool managedMemorySupported = true;
|
||||
cudaDeviceProp prop;
|
||||
std::vector<cudaDeviceProp> cudaProp;
|
||||
cudaDeviceConfig.GetCudaDeviceProp(cudaProp);
|
||||
for (int i = 0; i < numDevices && managedMemorySupported; ++i)
|
||||
{
|
||||
VTKM_CUDA_CALL(cudaGetDeviceProperties(&prop, i));
|
||||
// We check for concurrentManagedAccess, as devices with only the
|
||||
// managedAccess property have extra synchronization requirements.
|
||||
managedMemorySupported = managedMemorySupported && prop.concurrentManagedAccess;
|
||||
managedMemorySupported = managedMemorySupported && cudaProp[i].concurrentManagedAccess;
|
||||
}
|
||||
|
||||
HardwareSupportsManagedMemory = managedMemorySupported;
|
||||
|
@ -13,6 +13,15 @@
|
||||
#include <vtkm/cont/cuda/internal/DeviceAdapterTagCuda.h>
|
||||
#include <vtkm/cont/internal/RuntimeDeviceConfiguration.h>
|
||||
|
||||
#include <vtkm/cont/Logging.h>
|
||||
#include <vtkm/cont/cuda/ErrorCuda.h>
|
||||
|
||||
VTKM_THIRDPARTY_PRE_INCLUDE
|
||||
#include <cuda.h>
|
||||
VTKM_THIRDPARTY_POST_INCLUDE
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace vtkm
|
||||
{
|
||||
namespace cont
|
||||
@ -24,27 +33,76 @@ template <>
|
||||
class RuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>
|
||||
: public vtkm::cont::internal::RuntimeDeviceConfigurationBase
|
||||
{
|
||||
public:
|
||||
RuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>()
|
||||
{
|
||||
int tmp;
|
||||
VTKM_CUDA_CALL(cudaGetDeviceCount(&tmp));
|
||||
this->CudaDeviceCount = tmp;
|
||||
this->CudaProp.resize(this->CudaDeviceCount);
|
||||
for (int i = 0; i < this->CudaDeviceCount; ++i)
|
||||
{
|
||||
VTKM_CUDA_CALL(cudaGetDeviceProperties(&this->CudaProp[i], i));
|
||||
}
|
||||
}
|
||||
|
||||
VTKM_CONT vtkm::cont::DeviceAdapterId GetDevice() const override final
|
||||
{
|
||||
return vtkm::cont::DeviceAdapterTagCuda{};
|
||||
}
|
||||
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetDeviceInstance(const vtkm::Id&) override final
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode SetDeviceInstance(
|
||||
const vtkm::Id& value) override final
|
||||
{
|
||||
// TODO: set the cuda device instance
|
||||
if (value >= this->CudaDeviceCount)
|
||||
{
|
||||
VTKM_LOG_S(
|
||||
vtkm::cont::LogLevel::Error,
|
||||
"Failed to set CudaDeviceInstance, supplied id exceeds the number of available devices: "
|
||||
<< value << " >= " << this->CudaDeviceCount);
|
||||
return RuntimeDeviceConfigReturnCode::INVALID_VALUE;
|
||||
}
|
||||
try
|
||||
{
|
||||
VTKM_CUDA_CALL(cudaSetDevice(value));
|
||||
return RuntimeDeviceConfigReturnCode::SUCCESS;
|
||||
}
|
||||
catch (const vtkm::cont::cuda::ErrorCuda& err)
|
||||
{
|
||||
VTKM_LOG_S(vtkm::cont::LogLevel::Error,
|
||||
"Failed to set CudaDeviceInstance: " << err.GetMessage());
|
||||
return RuntimeDeviceConfigReturnCode::INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance(
|
||||
vtkm::Id& value) const override final
|
||||
{
|
||||
int tmp;
|
||||
VTKM_CUDA_CALL(cudaGetDevice(&tmp));
|
||||
value = tmp;
|
||||
return RuntimeDeviceConfigReturnCode::SUCCESS;
|
||||
}
|
||||
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetDeviceInstance(vtkm::Id&) const override final
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetMaxDevices(
|
||||
vtkm::Id& value) const override final
|
||||
{
|
||||
// TODO: Get the cuda device instance (also maybe a list of available devices?)
|
||||
value = this->CudaDeviceCount;
|
||||
return RuntimeDeviceConfigReturnCode::SUCCESS;
|
||||
}
|
||||
|
||||
VTKM_CONT virtual RuntimeDeviceConfigReturnCode GetMaxDevices(vtkm::Id&) const override final
|
||||
/// A function only available for use by the Cuda instance of this class
|
||||
/// Used to grab the CudaDeviceProp structs for all available devices
|
||||
VTKM_CONT RuntimeDeviceConfigReturnCode
|
||||
GetCudaDeviceProp(std::vector<cudaDeviceProp>& value) const
|
||||
{
|
||||
value = CudaProp;
|
||||
return RuntimeDeviceConfigReturnCode::SUCCESS;
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<cudaDeviceProp> CudaProp;
|
||||
vtkm::Id CudaDeviceCount;
|
||||
};
|
||||
} // namespace vtkm::cont::internal
|
||||
} // namespace vtkm::cont
|
||||
|
@ -26,6 +26,7 @@ set(unit_tests
|
||||
UnitTestCudaMathEdgeCases.cu
|
||||
UnitTestCudaShareUserProvidedManagedMemory.cu
|
||||
UnitTestCudaPointLocatorSparseGrid.cu
|
||||
UnitTestCudaRuntimeDeviceConfiguration.cu
|
||||
)
|
||||
|
||||
if (NOT VTKm_NO_DEPRECATED_VIRTUAL)
|
||||
|
@ -0,0 +1,64 @@
|
||||
//============================================================================
|
||||
// Copyright (c) Kitware, Inc.
|
||||
// All rights reserved.
|
||||
// See LICENSE.txt for details.
|
||||
//
|
||||
// This software is distributed WITHOUT ANY WARRANTY; without even
|
||||
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
|
||||
// PURPOSE. See the above copyright notice for more information.
|
||||
//============================================================================
|
||||
#include <vtkm/cont/RuntimeDeviceTracker.h>
|
||||
#include <vtkm/cont/cuda/DeviceAdapterCuda.h>
|
||||
#include <vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h>
|
||||
|
||||
namespace internal = vtkm::cont::internal;
|
||||
|
||||
namespace vtkm
|
||||
{
|
||||
namespace cont
|
||||
{
|
||||
namespace testing
|
||||
{
|
||||
|
||||
template <>
|
||||
VTKM_CONT void
|
||||
TestingRuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>::TestRuntimeConfig()
|
||||
{
|
||||
auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions();
|
||||
int numDevices = 0;
|
||||
VTKM_CUDA_CALL(cudaGetDeviceCount(&numDevices));
|
||||
vtkm::Id selectedDevice = numDevices > 0 ? numDevices - 1 : 0;
|
||||
deviceOptions.VTKmDeviceInstance.SetOption(selectedDevice);
|
||||
auto& config =
|
||||
RuntimeDeviceInformation{}.GetRuntimeConfiguration(DeviceAdapterTagCuda(), deviceOptions);
|
||||
vtkm::Id setDevice;
|
||||
VTKM_TEST_ASSERT(config.GetDeviceInstance(setDevice) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get device instance");
|
||||
VTKM_TEST_ASSERT(setDevice == selectedDevice,
|
||||
"RTC's setDevice != selectedDevice cuda direct! " + std::to_string(setDevice) +
|
||||
" != " + std::to_string(selectedDevice));
|
||||
vtkm::Id maxDevices;
|
||||
VTKM_TEST_ASSERT(config.GetMaxDevices(maxDevices) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get max devices");
|
||||
VTKM_TEST_ASSERT(maxDevices == numDevices,
|
||||
"RTC's maxDevices != numDevices cuda direct! " + std::to_string(maxDevices) +
|
||||
" != " + std::to_string(numDevices));
|
||||
std::vector<cudaDeviceProp> cudaProps;
|
||||
dynamic_cast<internal::RuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagCuda>&>(config)
|
||||
.GetCudaDeviceProp(cudaProps);
|
||||
VTKM_TEST_ASSERT(maxDevices == static_cast<vtkm::Id>(cudaProps.size()),
|
||||
"CudaProp's size != maxDevices! " + std::to_string(cudaProps.size()) +
|
||||
" != " + std::to_string(maxDevices));
|
||||
}
|
||||
|
||||
} // namespace vtkm::cont::testing
|
||||
} // namespace vtkm::cont
|
||||
} // namespace vtkm
|
||||
|
||||
int UnitTestCudaRuntimeDeviceConfiguration(int argc, char* argv[])
|
||||
{
|
||||
return vtkm::cont::testing::TestingRuntimeDeviceConfiguration<
|
||||
vtkm::cont::DeviceAdapterTagCuda>::Run(argc, argv);
|
||||
}
|
@ -24,13 +24,14 @@ namespace cont
|
||||
namespace internal
|
||||
{
|
||||
|
||||
enum class RuntimeDeviceConfigReturnCode
|
||||
enum class RuntimeDeviceConfigReturnCode : vtkm::Id
|
||||
{
|
||||
SUCCESS,
|
||||
OUT_OF_BOUNDS,
|
||||
INVALID_FOR_DEVICE,
|
||||
INVALID_VALUE,
|
||||
NOT_APPLIED
|
||||
NOT_APPLIED,
|
||||
INTERNAL_ERROR
|
||||
};
|
||||
|
||||
class VTKM_CONT_EXPORT RuntimeDeviceConfigurationBase
|
||||
|
@ -11,6 +11,7 @@
|
||||
#define vtk_m_cont_kokkos_internal_RuntimeDeviceConfigurationKokkos_h
|
||||
|
||||
#include <vtkm/cont/ErrorInternal.h>
|
||||
#include <vtkm/cont/Logging.h>
|
||||
#include <vtkm/cont/internal/RuntimeDeviceConfiguration.h>
|
||||
#include <vtkm/cont/kokkos/internal/DeviceAdapterTagKokkos.h>
|
||||
|
||||
|
@ -11,9 +11,102 @@
|
||||
#include <vtkm/cont/kokkos/DeviceAdapterKokkos.h>
|
||||
#include <vtkm/cont/testing/TestingRuntimeDeviceConfiguration.h>
|
||||
|
||||
namespace internal = vtkm::cont::internal;
|
||||
|
||||
namespace vtkm
|
||||
{
|
||||
namespace cont
|
||||
{
|
||||
namespace testing
|
||||
{
|
||||
|
||||
template <>
|
||||
VTKM_CONT void
|
||||
TestingRuntimeDeviceConfiguration<vtkm::cont::DeviceAdapterTagKokkos>::TestRuntimeConfig()
|
||||
{
|
||||
int argc;
|
||||
char** argv;
|
||||
vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-numa=4");
|
||||
vtkm::cont::testing::Testing::SetEnv("KOKKOS_DEVICE_ID", "0");
|
||||
auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions();
|
||||
bool threw = false;
|
||||
try
|
||||
{
|
||||
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
DeviceAdapterTagKokkos(), deviceOptions, argc, argv);
|
||||
}
|
||||
catch (const std::runtime_error& e)
|
||||
{
|
||||
threw = true;
|
||||
}
|
||||
VTKM_TEST_ASSERT(threw,
|
||||
"GetRuntimeConfiguration should have thrown, env KOKKOS_DEVICE_ID didn't match");
|
||||
VTKM_TEST_ASSERT(!Kokkos::is_initialized(), "Kokkos should not be initialized at this point");
|
||||
deviceOptions.VTKmDeviceInstance.SetOption(0);
|
||||
internal::RuntimeDeviceConfigurationBase& config =
|
||||
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
DeviceAdapterTagKokkos(), deviceOptions, argc, argv);
|
||||
VTKM_TEST_ASSERT(Kokkos::is_initialized(), "Kokkos should be initialized at this point");
|
||||
|
||||
// Test that args are set and the right arg priority is applied
|
||||
vtkm::Id testValue;
|
||||
VTKM_TEST_ASSERT(config.GetThreads(testValue) == internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set threads");
|
||||
VTKM_TEST_ASSERT(testValue == 8,
|
||||
"Set threads does not match expected value: 8 != " + std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set numa regions");
|
||||
VTKM_TEST_ASSERT(testValue == 4,
|
||||
"Set numa regions does not match expected value: 4 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set device instance");
|
||||
VTKM_TEST_ASSERT(testValue == 0,
|
||||
"Set device instance does not match expected value: 0 != " +
|
||||
std::to_string(testValue));
|
||||
// Ensure that with kokkos we can't re-initialize or set values after the first initialize
|
||||
// Should pop up a few warnings in the test logs
|
||||
deviceOptions.VTKmNumThreads.SetOption(16);
|
||||
deviceOptions.VTKmNumaRegions.SetOption(2);
|
||||
deviceOptions.VTKmDeviceInstance.SetOption(5);
|
||||
config.Initialize(deviceOptions);
|
||||
VTKM_TEST_ASSERT(config.SetThreads(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set threads after kokkos is initalized");
|
||||
VTKM_TEST_ASSERT(config.SetNumaRegions(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set numa regions after kokkos is initalized");
|
||||
VTKM_TEST_ASSERT(config.SetDeviceInstance(1) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set device instnace after kokkos is initalized");
|
||||
|
||||
// make sure all the values are the same
|
||||
VTKM_TEST_ASSERT(config.GetThreads(testValue) == internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set threads");
|
||||
VTKM_TEST_ASSERT(testValue == 8,
|
||||
"Set threads does not match expected value: 8 != " + std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set numa regions");
|
||||
VTKM_TEST_ASSERT(testValue == 4,
|
||||
"Set numa regions does not match expected value: 4 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set device instance");
|
||||
VTKM_TEST_ASSERT(testValue == 0,
|
||||
"Set device instance does not match expected value: 0 != " +
|
||||
std::to_string(testValue));
|
||||
|
||||
vtkm::cont::testing::Testing::UnsetEnv("KOKKOS_DEVICE_ID");
|
||||
}
|
||||
|
||||
} // namespace vtkm::cont::testing
|
||||
} // namespace vtkm::cont
|
||||
} // namespace vtkm
|
||||
|
||||
int UnitTestKokkosRuntimeDeviceConfiguration(int argc, char* argv[])
|
||||
{
|
||||
vtkm::cont::GetRuntimeDeviceTracker().ForceDevice(vtkm::cont::DeviceAdapterTagKokkos{});
|
||||
return vtkm::cont::testing::TestingRuntimeDeviceConfiguration<
|
||||
vtkm::cont::DeviceAdapterTagKokkos>::Run(argc, argv);
|
||||
}
|
||||
|
@ -43,137 +43,18 @@ struct TestingRuntimeDeviceConfiguration
|
||||
return runtimeDeviceOptions;
|
||||
}
|
||||
|
||||
VTKM_CONT
|
||||
static void TestSerial() {}
|
||||
|
||||
VTKM_CONT
|
||||
static void TestTBB() {}
|
||||
|
||||
VTKM_CONT
|
||||
static void TestOpenMP() {}
|
||||
|
||||
VTKM_CONT
|
||||
static void TestCuda() {}
|
||||
|
||||
VTKM_CONT
|
||||
static void TestKokkos()
|
||||
{
|
||||
int argc;
|
||||
char** argv;
|
||||
vtkm::cont::testing::Testing::MakeArgs(argc, argv, "--kokkos-numa=4");
|
||||
vtkm::cont::testing::Testing::SetEnv("KOKKOS_DEVICE_ID", "0");
|
||||
auto deviceOptions = TestingRuntimeDeviceConfiguration::DefaultInitializeConfigOptions();
|
||||
bool threw = false;
|
||||
try
|
||||
{
|
||||
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
DeviceAdapterTag(), deviceOptions, argc, argv);
|
||||
}
|
||||
catch (const std::runtime_error& e)
|
||||
{
|
||||
threw = true;
|
||||
}
|
||||
VTKM_TEST_ASSERT(
|
||||
threw, "GetRuntimeConfiguration should have thrown, env KOKKOS_DEVICE_ID didn't match");
|
||||
VTKM_TEST_ASSERT(!Kokkos::is_initialized(), "Kokkos should not be initialized at this point");
|
||||
deviceOptions.VTKmDeviceInstance.SetOption(0);
|
||||
internal::RuntimeDeviceConfigurationBase& config =
|
||||
RuntimeDeviceInformation{}.GetRuntimeConfiguration(
|
||||
DeviceAdapterTag(), deviceOptions, argc, argv);
|
||||
VTKM_TEST_ASSERT(Kokkos::is_initialized(), "Kokkos should be initialized at this point");
|
||||
|
||||
// Test that args are set and the right arg priority is applied
|
||||
vtkm::Id testValue;
|
||||
VTKM_TEST_ASSERT(config.GetThreads(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set threads");
|
||||
VTKM_TEST_ASSERT(testValue == 8,
|
||||
"Set threads does not match expected value: 8 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set numa regions");
|
||||
VTKM_TEST_ASSERT(testValue == 4,
|
||||
"Set numa regions does not match expected value: 4 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set device instance");
|
||||
VTKM_TEST_ASSERT(testValue == 0,
|
||||
"Set device instance does not match expected value: 0 != " +
|
||||
std::to_string(testValue));
|
||||
|
||||
// Ensure that with kokkos we can't re-initialize or set values after the first initialize
|
||||
// Should pop up a few warnings in the test logs
|
||||
deviceOptions.VTKmNumThreads.SetOption(16);
|
||||
deviceOptions.VTKmNumaRegions.SetOption(2);
|
||||
deviceOptions.VTKmDeviceInstance.SetOption(5);
|
||||
config.Initialize(deviceOptions);
|
||||
VTKM_TEST_ASSERT(config.SetThreads(1) == internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set threads after kokkos is initalized");
|
||||
VTKM_TEST_ASSERT(config.SetNumaRegions(1) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set numa regions after kokkos is initalized");
|
||||
VTKM_TEST_ASSERT(config.SetDeviceInstance(1) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::NOT_APPLIED,
|
||||
"Shouldn't be able to set device instnace after kokkos is initalized");
|
||||
|
||||
// make sure all the values are the same
|
||||
VTKM_TEST_ASSERT(config.GetThreads(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set threads");
|
||||
VTKM_TEST_ASSERT(testValue == 8,
|
||||
"Set threads does not match expected value: 8 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetNumaRegions(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set numa regions");
|
||||
VTKM_TEST_ASSERT(testValue == 4,
|
||||
"Set numa regions does not match expected value: 4 != " +
|
||||
std::to_string(testValue));
|
||||
VTKM_TEST_ASSERT(config.GetDeviceInstance(testValue) ==
|
||||
internal::RuntimeDeviceConfigReturnCode::SUCCESS,
|
||||
"Failed to get set device instance");
|
||||
VTKM_TEST_ASSERT(testValue == 0,
|
||||
"Set device instance does not match expected value: 0 != " +
|
||||
std::to_string(testValue));
|
||||
|
||||
vtkm::cont::testing::Testing::UnsetEnv("KOKKOS_DEVICE_ID");
|
||||
}
|
||||
VTKM_CONT static void TestRuntimeConfig(){};
|
||||
|
||||
struct TestRunner
|
||||
{
|
||||
VTKM_CONT
|
||||
void operator()() const
|
||||
{
|
||||
switch (DeviceAdapterTag{}.GetValue())
|
||||
{
|
||||
case vtkm::cont::DeviceAdapterTagSerial{}.GetValue():
|
||||
TestingRuntimeDeviceConfiguration::TestSerial();
|
||||
break;
|
||||
case vtkm::cont::DeviceAdapterTagTBB{}.GetValue():
|
||||
TestingRuntimeDeviceConfiguration::TestTBB();
|
||||
break;
|
||||
case vtkm::cont::DeviceAdapterTagOpenMP{}.GetValue():
|
||||
TestingRuntimeDeviceConfiguration::TestOpenMP();
|
||||
break;
|
||||
case vtkm::cont::DeviceAdapterTagCuda{}.GetValue():
|
||||
TestingRuntimeDeviceConfiguration::TestCuda();
|
||||
break;
|
||||
case vtkm::cont::DeviceAdapterTagKokkos{}.GetValue():
|
||||
TestingRuntimeDeviceConfiguration::TestKokkos();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
void operator()() const { TestRuntimeConfig(); }
|
||||
};
|
||||
|
||||
static VTKM_CONT int Run(int, char*[])
|
||||
{
|
||||
// For the Kokkos version of this test we don't not want Initialize to be called
|
||||
// so we directly execute the testing functor instead of calling Run
|
||||
vtkm::cont::GetRuntimeDeviceTracker().ForceDevice(DeviceAdapterTag());
|
||||
return vtkm::cont::testing::Testing::ExecuteFunction(TestRunner{});
|
||||
}
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user