diff --git a/vtkm/CMakeLists.txt b/vtkm/CMakeLists.txt index 436d07569..cafea8585 100644 --- a/vtkm/CMakeLists.txt +++ b/vtkm/CMakeLists.txt @@ -67,11 +67,7 @@ vtkm_declare_headers(${headers}) #----------------------------------------------------------------------------- #first add all the components vtkm that are shared between control and exec -if(VTKm_ENABLE_MPI) - # This `if` is temporary and will be removed once `diy` supports building - # without MPI. - add_subdirectory(thirdparty/diy) -endif() +add_subdirectory(thirdparty/diy) add_subdirectory(testing) add_subdirectory(internal) diff --git a/vtkm/cont/AssignerMultiBlock.cxx b/vtkm/cont/AssignerMultiBlock.cxx index ecb34ce8f..efb7ee70b 100644 --- a/vtkm/cont/AssignerMultiBlock.cxx +++ b/vtkm/cont/AssignerMultiBlock.cxx @@ -19,10 +19,10 @@ //============================================================================ #include -#if defined(VTKM_ENABLE_MPI) +#include +#include // clang-format off -#include #include VTKM_DIY(diy/mpi.hpp) // clang-format on @@ -76,5 +76,3 @@ int AssignerMultiBlock::rank(int gid) const } // vtkm::cont } // vtkm - -#endif // defined(VTKM_ENABLE_MPI) diff --git a/vtkm/cont/AssignerMultiBlock.h b/vtkm/cont/AssignerMultiBlock.h index bbd32a513..6fd6e2d8a 100644 --- a/vtkm/cont/AssignerMultiBlock.h +++ b/vtkm/cont/AssignerMultiBlock.h @@ -20,13 +20,15 @@ #ifndef vtk_m_cont_AssignerMultiBlock_h #define vtk_m_cont_AssignerMultiBlock_h -#include +#include -#if defined(VTKM_ENABLE_MPI) -#include +#include +#include +#include + +#include // clang-format off -#include #include VTKM_DIY(diy/assigner.hpp) // clang-format on @@ -35,6 +37,8 @@ namespace vtkm namespace cont { +class MultiBlock; + /// \brief Assigner for `MultiBlock` blocks. /// /// `AssignerMultiBlock` is a `diy::Assigner` implementation that uses @@ -70,5 +74,4 @@ private: } } -#endif // defined(VTKM_ENABLE_MPI) #endif diff --git a/vtkm/cont/CMakeLists.txt b/vtkm/cont/CMakeLists.txt index 993737b91..7088a3743 100644 --- a/vtkm/cont/CMakeLists.txt +++ b/vtkm/cont/CMakeLists.txt @@ -155,15 +155,12 @@ vtkm_library( WRAP_FOR_CUDA ${device_sources} HEADERS ${header_impls}) target_include_directories(vtkm_cont PRIVATE ${VTKm_BACKEND_INCLUDE_DIRS}) +target_link_libraries(vtkm_cont PUBLIC vtkm_diy) if(VTKm_ENABLE_CUDA) add_dependencies(vtkm_cont vtkm_cont_cuda) endif() -if(VTKm_ENABLE_MPI) - # This will become a required dependency eventually. - target_link_libraries(vtkm_cont PUBLIC diy) -endif() #----------------------------------------------------------------------------- add_subdirectory(testing) diff --git a/vtkm/cont/DecomposerMultiBlock.h b/vtkm/cont/DecomposerMultiBlock.h index 342968d83..58c932352 100644 --- a/vtkm/cont/DecomposerMultiBlock.h +++ b/vtkm/cont/DecomposerMultiBlock.h @@ -20,9 +20,14 @@ #ifndef vtk_m_cont_DecomposerMultiBlock_h #define vtk_m_cont_DecomposerMultiBlock_h -#include -#if defined(VTKM_ENABLE_MPI) -#include +#include + +#include +#include + +// clang-format off +#include VTKM_DIY(diy/assigner.hpp) +// clang-format on namespace vtkm { @@ -52,6 +57,4 @@ public: } } -#endif // defined(VTKM_ENABLE_MPI) - #endif diff --git a/vtkm/cont/EnvironmentTracker.cxx b/vtkm/cont/EnvironmentTracker.cxx index 9767d90bc..956ce5dbe 100644 --- a/vtkm/cont/EnvironmentTracker.cxx +++ b/vtkm/cont/EnvironmentTracker.cxx @@ -19,30 +19,14 @@ //============================================================================ #include -#if defined(VTKM_ENABLE_MPI) - // clang-format off -#include #include VTKM_DIY(diy/mpi.hpp) // clang-format on -#else -namespace diy -{ -namespace mpi -{ -class communicator -{ -}; -} -} -#endif - namespace vtkm { namespace cont { -#if defined(VTKM_ENABLE_MPI) namespace internal { static diy::mpi::communicator GlobalCommuncator(MPI_COMM_NULL); @@ -57,16 +41,5 @@ const diy::mpi::communicator& EnvironmentTracker::GetCommunicator() { return vtkm::cont::internal::GlobalCommuncator; } -#else -void EnvironmentTracker::SetCommunicator(const diy::mpi::communicator&) -{ -} - -const diy::mpi::communicator& EnvironmentTracker::GetCommunicator() -{ - static diy::mpi::communicator tmp; - return tmp; -} -#endif } // namespace vtkm::cont } // namespace vtkm diff --git a/vtkm/cont/EnvironmentTracker.h b/vtkm/cont/EnvironmentTracker.h index d26785abc..1f82055c5 100644 --- a/vtkm/cont/EnvironmentTracker.h +++ b/vtkm/cont/EnvironmentTracker.h @@ -22,13 +22,8 @@ #include #include -#include #include - -#if defined(VTKM_ENABLE_MPI) -// needed for diy mangling. #include -#endif namespace diy { @@ -42,6 +37,11 @@ namespace vtkm { namespace cont { + +/// \brief Maintain MPI controller, if any, for distributed operation. +/// +/// `EnvironmentTracker` is a class that provides static API to track the global +/// MPI controller to use for operating in a distributed environment. class VTKM_CONT_EXPORT EnvironmentTracker { public: diff --git a/vtkm/cont/MultiBlock.cxx b/vtkm/cont/MultiBlock.cxx index 4323b9bba..9ebb245a8 100644 --- a/vtkm/cont/MultiBlock.cxx +++ b/vtkm/cont/MultiBlock.cxx @@ -31,7 +31,6 @@ #include #include -#if defined(VTKM_ENABLE_MPI) // clang-format off #include #include VTKM_DIY(diy/decomposition.hpp) @@ -72,8 +71,6 @@ const vtkm::cont::DataSet& GetBlock(const vtkm::cont::MultiBlock& mb, } } -#endif - namespace vtkm { namespace cont @@ -137,7 +134,6 @@ vtkm::Id MultiBlock::GetNumberOfBlocks() const VTKM_CONT vtkm::Id MultiBlock::GetGlobalNumberOfBlocks() const { -#if defined(VTKM_ENABLE_MPI) auto world = vtkm::cont::EnvironmentTracker::GetCommunicator(); const auto local_count = this->GetNumberOfBlocks(); @@ -151,9 +147,6 @@ vtkm::Id MultiBlock::GetGlobalNumberOfBlocks() const master.process_collectives(); vtkm::Id global_count = master.proxy(0).get(); return global_count; -#else - return this->GetNumberOfBlocks(); -#endif } VTKM_CONT @@ -207,7 +200,6 @@ void MultiBlock::ReplaceBlock(vtkm::Id index, vtkm::cont::DataSet& ds) VTKM_CONT vtkm::Bounds MultiBlock::GetBounds(vtkm::Id coordinate_system_index) const { -#if defined(VTKM_ENABLE_MPI) auto world = vtkm::cont::EnvironmentTracker::GetCommunicator(); diy::Master master(world, 1, @@ -259,19 +251,6 @@ VTKM_CONT vtkm::Bounds MultiBlock::GetBounds(vtkm::Id coordinate_system_index) c return (*master.block(0)); } return vtkm::Bounds(); - -#else - const vtkm::Id index = coordinate_system_index; - const size_t num_blocks = this->Blocks.size(); - - vtkm::Bounds bounds; - for (size_t i = 0; i < num_blocks; ++i) - { - vtkm::Bounds block_bounds = this->GetBlockBounds(i, index); - bounds.Include(block_bounds); - } - return bounds; -#endif } VTKM_CONT vtkm::Bounds MultiBlock::GetBlockBounds(const std::size_t& block_index, @@ -305,7 +284,6 @@ VTKM_CONT vtkm::cont::ArrayHandle MultiBlock::GetGlobalRange(const VTKM_CONT vtkm::cont::ArrayHandle MultiBlock::GetGlobalRange( const std::string& field_name) const { -#if defined(VTKM_ENABLE_MPI) using BlockMetaData = std::vector; auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator(); @@ -366,64 +344,6 @@ VTKM_CONT vtkm::cont::ArrayHandle MultiBlock::GetGlobalRange( vtkm::cont::ArrayHandle range; vtkm::cont::ArrayCopy(vtkm::cont::make_ArrayHandle(ranges), range); return range; -#else - bool valid_field = true; - const size_t num_blocks = this->Blocks.size(); - - vtkm::cont::ArrayHandle range; - vtkm::Id num_components = 0; - - for (size_t i = 0; i < num_blocks; ++i) - { - if (!this->Blocks[i].HasField(field_name)) - { - valid_field = false; - break; - } - - const vtkm::cont::Field& field = this->Blocks[i].GetField(field_name); - vtkm::cont::ArrayHandle sub_range = field.GetRange(); - - vtkm::cont::ArrayHandle::PortalConstControl sub_range_control = - sub_range.GetPortalConstControl(); - vtkm::cont::ArrayHandle::PortalControl range_control = range.GetPortalControl(); - - if (i == 0) - { - num_components = sub_range_control.GetNumberOfValues(); - range = sub_range; - continue; - } - - vtkm::Id components = sub_range_control.GetNumberOfValues(); - - if (components != num_components) - { - std::stringstream msg; - msg << "GetRange call failed. The number of components (" << components << ") in field " - << field_name << " from block " << i << " does not match the number of components " - << "(" << num_components << ") in block 0"; - throw ErrorExecution(msg.str()); - } - - for (vtkm::Id c = 0; c < components; ++c) - { - vtkm::Range s_range = sub_range_control.Get(c); - vtkm::Range c_range = range_control.Get(c); - c_range.Include(s_range); - range_control.Set(c, c_range); - } - } - - if (!valid_field) - { - std::string msg = "GetRange call failed. "; - msg += " Field " + field_name + " did not exist in at least one block."; - throw ErrorExecution(msg); - } - - return range; -#endif } VTKM_CONT diff --git a/vtkm/cont/testing/UnitTestMultiBlock.cxx b/vtkm/cont/testing/UnitTestMultiBlock.cxx index 460d9fd59..710f6783f 100644 --- a/vtkm/cont/testing/UnitTestMultiBlock.cxx +++ b/vtkm/cont/testing/UnitTestMultiBlock.cxx @@ -34,16 +34,12 @@ #include #include #include - -#if defined(VTKM_ENABLE_MPI) +#include // clang-format off -#include #include VTKM_DIY(diy/master.hpp) // clang-format on -#endif - void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& RightDateSet); static void MultiBlockTest() { @@ -56,10 +52,7 @@ static void MultiBlockTest() multiblock.AddBlock(TDset1); multiblock.AddBlock(TDset2); - int procsize = 1; -#if defined(VTKM_ENABLE_MPI) - procsize = vtkm::cont::EnvironmentTracker::GetCommunicator().size(); -#endif + const int procsize = vtkm::cont::EnvironmentTracker::GetCommunicator().size(); VTKM_TEST_ASSERT(multiblock.GetNumberOfBlocks() == 2, "Incorrect number of blocks"); VTKM_TEST_ASSERT(multiblock.GetGlobalNumberOfBlocks() == 2 * procsize, @@ -160,7 +153,6 @@ static void MultiBlockTest() void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& RightDateSet) { - for (vtkm::Id j = 0; j < LeftDateSet.GetNumberOfFields(); j++) { vtkm::cont::ArrayHandle LDataArray; @@ -174,11 +166,7 @@ void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& Righ int UnitTestMultiBlock(int argc, char* argv[]) { - (void)argc; - (void)argv; -#if defined(VTKM_ENABLE_MPI) diy::mpi::environment env(argc, argv); vtkm::cont::EnvironmentTracker::SetCommunicator(diy::mpi::communicator(MPI_COMM_WORLD)); -#endif return vtkm::cont::testing::Testing::Run(MultiBlockTest); } diff --git a/vtkm/thirdparty/diy/CMakeLists.txt b/vtkm/thirdparty/diy/CMakeLists.txt index 9d84d87b4..3c1f570ec 100644 --- a/vtkm/thirdparty/diy/CMakeLists.txt +++ b/vtkm/thirdparty/diy/CMakeLists.txt @@ -19,45 +19,44 @@ ## this software. ## ##============================================================================= -add_library(diy INTERFACE) +add_library(vtkm_diy INTERFACE) vtkm_get_kit_name(kit_name kit_dir) # diy needs C++11 -target_compile_features(diy INTERFACE cxx_auto_type) +target_compile_features(vtkm_diy INTERFACE cxx_auto_type) # placeholder to support external DIY set(VTKM_USE_EXTERNAL_DIY OFF) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Configure.h.in ${VTKm_BINARY_INCLUDE_DIR}/${kit_dir}/Configure.h) -target_include_directories(diy INTERFACE +target_include_directories(vtkm_diy INTERFACE $ $ $) -# presently, this dependency is required. Make it optional in the future. -set(arg) -foreach(apath IN LISTS MPI_C_INCLUDE_PATH MPI_CXX_INCLUDE_PATH) - list(APPEND arg $) -endforeach() -target_include_directories(diy INTERFACE ${arg}) - -target_link_libraries(diy INTERFACE - $ - $) - -if(MPI_C_COMPILE_DEFINITIONS) - target_compile_definitions(diy INTERFACE - $<$:${MPI_C_COMPILE_DEFINITIONS}>) -endif() -if(MPI_CXX_COMPILE_DEFNITIONS) - target_compile_definitions(diy INTERFACE - $<$:${MPI_CXX_COMPILE_DEFNITIONS>) +if(VTKm_ENABLE_MPI) + set(arg) + foreach(apath IN LISTS MPI_C_INCLUDE_PATH MPI_CXX_INCLUDE_PATH) + list(APPEND arg $) + endforeach() + list(REMOVE_DUPLICATES arg) + target_include_directories(vtkm_diy INTERFACE ${arg}) + target_link_libraries(vtkm_diy INTERFACE + $ + $) + if(MPI_C_COMPILE_DEFINITIONS) + target_compile_definitions(vtkm_diy INTERFACE + $<$:${MPI_C_COMPILE_DEFINITIONS}>) + endif() + if(MPI_CXX_COMPILE_DEFNITIONS) + target_compile_definitions(vtkm_diy INTERFACE + $<$:${MPI_CXX_COMPILE_DEFNITIONS>) + endif() endif() -install(TARGETS diy +install(TARGETS vtkm_diy EXPORT ${VTKm_EXPORT_NAME}) ## Install headers diff --git a/vtkm/thirdparty/diy/Configure.h.in b/vtkm/thirdparty/diy/Configure.h.in index 9aafe6e00..869e7f0b7 100644 --- a/vtkm/thirdparty/diy/Configure.h.in +++ b/vtkm/thirdparty/diy/Configure.h.in @@ -22,9 +22,21 @@ #ifndef vtkm_diy_h #define vtkm_diy_h +#include + /* Use the diy library configured for VTM-m. */ #cmakedefine01 VTKM_USE_EXTERNAL_DIY +/* Whether to use MPI support in DIY */ +#if !defined(VTKM_ENABLE_MPI) +# define DIY_NO_MPI +#endif + +/* initially, we disable DIY threads. + * once we've sorted out how DIY threads and vtkm work together + * we will make this configurable.*/ +#define DIY_NO_THREADS + #if VTKM_USE_EXTERNAL_DIY # define VTKM_DIY(header)
#else