Make DIY a required dependency.

DIY now depends on MPI optionally. Hence we no longer need to depend on
DIY optionally based on whether MPI was enabled. Update cmake and c++
code to always use DIY-based components.

DIY is built with MPI support if VTKm_ENABLE_MPI is ON.

vtkm/CMakeLists.txt

@@ -67,11 +67,7 @@ vtkm_declare_headers(${headers})
 
 #-----------------------------------------------------------------------------
 #first add all the components vtkm that are shared between control and exec
-if(VTKm_ENABLE_MPI)
+add_subdirectory(thirdparty/diy)
-  # This `if` is temporary and will be removed once `diy` supports building
-  # without MPI.
-  add_subdirectory(thirdparty/diy)
-endif()
 add_subdirectory(testing)
 add_subdirectory(internal)
 

vtkm/cont/AssignerMultiBlock.cxx

@@ -19,10 +19,10 @@
 //============================================================================
 #include <vtkm/cont/AssignerMultiBlock.h>
 
-#if defined(VTKM_ENABLE_MPI)
+#include <vtkm/cont/EnvironmentTracker.h>
+#include <vtkm/cont/MultiBlock.h>
 
 // clang-format off
-#include <vtkm/cont/EnvironmentTracker.h>
 #include VTKM_DIY(diy/mpi.hpp)
 // clang-format on
 
@@ -76,5 +76,3 @@ int AssignerMultiBlock::rank(int gid) const
 
 } // vtkm::cont
 } // vtkm
-
-#endif // defined(VTKM_ENABLE_MPI)

vtkm/cont/AssignerMultiBlock.h

@@ -20,13 +20,15 @@
 #ifndef vtk_m_cont_AssignerMultiBlock_h
 #define vtk_m_cont_AssignerMultiBlock_h
 
-#include <vtkm/internal/Configure.h>
+#include <vtkm/cont/vtkm_cont_export.h>
 
-#if defined(VTKM_ENABLE_MPI)
+#include <vtkm/Types.h>
-#include <vtkm/cont/MultiBlock.h>
+#include <vtkm/internal/ExportMacros.h>
+#include <vtkm/thirdparty/diy/Configure.h>
+
+#include <vector>
 
 // clang-format off
-#include <vtkm/thirdparty/diy/Configure.h>
 #include VTKM_DIY(diy/assigner.hpp)
 // clang-format on
 
@@ -35,6 +37,8 @@ namespace vtkm
 namespace cont
 {
 
+class MultiBlock;
+
 /// \brief Assigner for `MultiBlock` blocks.
 ///
 /// `AssignerMultiBlock` is a `diy::Assigner` implementation that uses
@@ -70,5 +74,4 @@ private:
 }
 }
 
-#endif // defined(VTKM_ENABLE_MPI)
 #endif

vtkm/cont/CMakeLists.txt

@@ -155,15 +155,12 @@ vtkm_library(
   WRAP_FOR_CUDA ${device_sources}
   HEADERS ${header_impls})
 target_include_directories(vtkm_cont PRIVATE ${VTKm_BACKEND_INCLUDE_DIRS})
+target_link_libraries(vtkm_cont PUBLIC vtkm_diy)
 
 if(VTKm_ENABLE_CUDA)
   add_dependencies(vtkm_cont vtkm_cont_cuda)
 endif()
 
-if(VTKm_ENABLE_MPI)
-  # This will become a required dependency eventually.
-  target_link_libraries(vtkm_cont PUBLIC diy)
-endif()
 
 #-----------------------------------------------------------------------------
 add_subdirectory(testing)

vtkm/cont/DecomposerMultiBlock.h

@@ -20,9 +20,14 @@
 #ifndef vtk_m_cont_DecomposerMultiBlock_h
 #define vtk_m_cont_DecomposerMultiBlock_h
 
-#include <vtkm/internal/Configure.h>
+#include <vtkm/cont/vtkm_cont_export.h>
-#if defined(VTKM_ENABLE_MPI)
+
-#include <vtkm/cont/AssignerMultiBlock.h>
+#include <vtkm/internal/ExportMacros.h>
+#include <vtkm/thirdparty/diy/Configure.h>
+
+// clang-format off
+#include VTKM_DIY(diy/assigner.hpp)
+// clang-format on
 
 namespace vtkm
 {
@@ -52,6 +57,4 @@ public:
 }
 }
 
-#endif // defined(VTKM_ENABLE_MPI)
-
 #endif

vtkm/cont/EnvironmentTracker.cxx

@@ -19,30 +19,14 @@
 //============================================================================
 #include <vtkm/cont/EnvironmentTracker.h>
 
-#if defined(VTKM_ENABLE_MPI)
-
 // clang-format off
-#include <vtkm/thirdparty/diy/Configure.h>
 #include VTKM_DIY(diy/mpi.hpp)
 // clang-format on
 
-#else
-namespace diy
-{
-namespace mpi
-{
-class communicator
-{
-};
-}
-}
-#endif
-
 namespace vtkm
 {
 namespace cont
 {
-#if defined(VTKM_ENABLE_MPI)
 namespace internal
 {
 static diy::mpi::communicator GlobalCommuncator(MPI_COMM_NULL);
@@ -57,16 +41,5 @@ const diy::mpi::communicator& EnvironmentTracker::GetCommunicator()
 {
   return vtkm::cont::internal::GlobalCommuncator;
 }
-#else
-void EnvironmentTracker::SetCommunicator(const diy::mpi::communicator&)
-{
-}
-
-const diy::mpi::communicator& EnvironmentTracker::GetCommunicator()
-{
-  static diy::mpi::communicator tmp;
-  return tmp;
-}
-#endif
 } // namespace vtkm::cont
 } // namespace vtkm

vtkm/cont/EnvironmentTracker.h

@@ -22,13 +22,8 @@
 
 #include <vtkm/Types.h>
 #include <vtkm/cont/vtkm_cont_export.h>
-#include <vtkm/internal/Configure.h>
 #include <vtkm/internal/ExportMacros.h>
-
-#if defined(VTKM_ENABLE_MPI)
-// needed for diy mangling.
 #include <vtkm/thirdparty/diy/Configure.h>
-#endif
 
 namespace diy
 {
@@ -42,6 +37,11 @@ namespace vtkm
 {
 namespace cont
 {
+
+/// \brief Maintain MPI controller, if any, for distributed operation.
+///
+/// `EnvironmentTracker` is a class that provides static API to track the global
+/// MPI controller to use for operating in a distributed environment.
 class VTKM_CONT_EXPORT EnvironmentTracker
 {
 public:

vtkm/cont/MultiBlock.cxx

@@ -31,7 +31,6 @@
 #include <vtkm/cont/Field.h>
 #include <vtkm/cont/MultiBlock.h>
 
-#if defined(VTKM_ENABLE_MPI)
 // clang-format off
 #include <vtkm/thirdparty/diy/Configure.h>
 #include VTKM_DIY(diy/decomposition.hpp)
@@ -72,8 +71,6 @@ const vtkm::cont::DataSet& GetBlock(const vtkm::cont::MultiBlock& mb,
 }
 }
 
-#endif
-
 namespace vtkm
 {
 namespace cont
@@ -137,7 +134,6 @@ vtkm::Id MultiBlock::GetNumberOfBlocks() const
 VTKM_CONT
 vtkm::Id MultiBlock::GetGlobalNumberOfBlocks() const
 {
-#if defined(VTKM_ENABLE_MPI)
   auto world = vtkm::cont::EnvironmentTracker::GetCommunicator();
   const auto local_count = this->GetNumberOfBlocks();
 
@@ -151,9 +147,6 @@ vtkm::Id MultiBlock::GetGlobalNumberOfBlocks() const
   master.process_collectives();
   vtkm::Id global_count = master.proxy(0).get<vtkm::Id>();
   return global_count;
-#else
-  return this->GetNumberOfBlocks();
-#endif
 }
 
 VTKM_CONT
@@ -207,7 +200,6 @@ void MultiBlock::ReplaceBlock(vtkm::Id index, vtkm::cont::DataSet& ds)
 
 VTKM_CONT vtkm::Bounds MultiBlock::GetBounds(vtkm::Id coordinate_system_index) const
 {
-#if defined(VTKM_ENABLE_MPI)
   auto world = vtkm::cont::EnvironmentTracker::GetCommunicator();
   diy::Master master(world,
                      1,
@@ -259,19 +251,6 @@ VTKM_CONT vtkm::Bounds MultiBlock::GetBounds(vtkm::Id coordinate_system_index) c
     return (*master.block<vtkm::Bounds>(0));
   }
   return vtkm::Bounds();
-
-#else
-  const vtkm::Id index = coordinate_system_index;
-  const size_t num_blocks = this->Blocks.size();
-
-  vtkm::Bounds bounds;
-  for (size_t i = 0; i < num_blocks; ++i)
-  {
-    vtkm::Bounds block_bounds = this->GetBlockBounds(i, index);
-    bounds.Include(block_bounds);
-  }
-  return bounds;
-#endif
 }
 
 VTKM_CONT vtkm::Bounds MultiBlock::GetBlockBounds(const std::size_t& block_index,
@@ -305,7 +284,6 @@ VTKM_CONT vtkm::cont::ArrayHandle<vtkm::Range> MultiBlock::GetGlobalRange(const
 VTKM_CONT vtkm::cont::ArrayHandle<vtkm::Range> MultiBlock::GetGlobalRange(
   const std::string& field_name) const
 {
-#if defined(VTKM_ENABLE_MPI)
   using BlockMetaData = std::vector<vtkm::Range>;
 
   auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator();
@@ -366,64 +344,6 @@ VTKM_CONT vtkm::cont::ArrayHandle<vtkm::Range> MultiBlock::GetGlobalRange(
   vtkm::cont::ArrayHandle<vtkm::Range> range;
   vtkm::cont::ArrayCopy(vtkm::cont::make_ArrayHandle(ranges), range);
   return range;
-#else
-  bool valid_field = true;
-  const size_t num_blocks = this->Blocks.size();
-
-  vtkm::cont::ArrayHandle<vtkm::Range> range;
-  vtkm::Id num_components = 0;
-
-  for (size_t i = 0; i < num_blocks; ++i)
-  {
-    if (!this->Blocks[i].HasField(field_name))
-    {
-      valid_field = false;
-      break;
-    }
-
-    const vtkm::cont::Field& field = this->Blocks[i].GetField(field_name);
-    vtkm::cont::ArrayHandle<vtkm::Range> sub_range = field.GetRange();
-
-    vtkm::cont::ArrayHandle<vtkm::Range>::PortalConstControl sub_range_control =
-      sub_range.GetPortalConstControl();
-    vtkm::cont::ArrayHandle<vtkm::Range>::PortalControl range_control = range.GetPortalControl();
-
-    if (i == 0)
-    {
-      num_components = sub_range_control.GetNumberOfValues();
-      range = sub_range;
-      continue;
-    }
-
-    vtkm::Id components = sub_range_control.GetNumberOfValues();
-
-    if (components != num_components)
-    {
-      std::stringstream msg;
-      msg << "GetRange call failed. The number of components (" << components << ") in field "
-          << field_name << " from block " << i << " does not match the number of components "
-          << "(" << num_components << ") in block 0";
-      throw ErrorExecution(msg.str());
-    }
-
-    for (vtkm::Id c = 0; c < components; ++c)
-    {
-      vtkm::Range s_range = sub_range_control.Get(c);
-      vtkm::Range c_range = range_control.Get(c);
-      c_range.Include(s_range);
-      range_control.Set(c, c_range);
-    }
-  }
-
-  if (!valid_field)
-  {
-    std::string msg = "GetRange call failed. ";
-    msg += " Field " + field_name + " did not exist in at least one block.";
-    throw ErrorExecution(msg);
-  }
-
-  return range;
-#endif
 }
 
 VTKM_CONT

vtkm/cont/testing/UnitTestMultiBlock.cxx

@@ -34,16 +34,12 @@
 #include <vtkm/cont/testing/MakeTestDataSet.h>
 #include <vtkm/cont/testing/Testing.h>
 #include <vtkm/exec/ConnectivityStructured.h>
-
+#include <vtkm/thirdparty/diy/Configure.h>
-#if defined(VTKM_ENABLE_MPI)
 
 // clang-format off
-#include <vtkm/thirdparty/diy/Configure.h>
 #include VTKM_DIY(diy/master.hpp)
 // clang-format on
 
-#endif
-
 void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& RightDateSet);
 static void MultiBlockTest()
 {
@@ -56,10 +52,7 @@ static void MultiBlockTest()
   multiblock.AddBlock(TDset1);
   multiblock.AddBlock(TDset2);
 
-  int procsize = 1;
+  const int procsize = vtkm::cont::EnvironmentTracker::GetCommunicator().size();
-#if defined(VTKM_ENABLE_MPI)
-  procsize = vtkm::cont::EnvironmentTracker::GetCommunicator().size();
-#endif
 
   VTKM_TEST_ASSERT(multiblock.GetNumberOfBlocks() == 2, "Incorrect number of blocks");
   VTKM_TEST_ASSERT(multiblock.GetGlobalNumberOfBlocks() == 2 * procsize,
@@ -160,7 +153,6 @@ static void MultiBlockTest()
 
 void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& RightDateSet)
 {
-
   for (vtkm::Id j = 0; j < LeftDateSet.GetNumberOfFields(); j++)
   {
     vtkm::cont::ArrayHandle<vtkm::Float32> LDataArray;
@@ -174,11 +166,7 @@ void DataSet_Compare(vtkm::cont::DataSet& LeftDateSet, vtkm::cont::DataSet& Righ
 
 int UnitTestMultiBlock(int argc, char* argv[])
 {
-  (void)argc;
-  (void)argv;
-#if defined(VTKM_ENABLE_MPI)
   diy::mpi::environment env(argc, argv);
   vtkm::cont::EnvironmentTracker::SetCommunicator(diy::mpi::communicator(MPI_COMM_WORLD));
-#endif
   return vtkm::cont::testing::Testing::Run(MultiBlockTest);
 }

vtkm/thirdparty/diy/CMakeLists.txt

@@ -19,45 +19,44 @@
 ##  this software.
 ##
 ##=============================================================================
-add_library(diy INTERFACE)
+add_library(vtkm_diy INTERFACE)
 
 vtkm_get_kit_name(kit_name kit_dir)
 
 # diy needs C++11
-target_compile_features(diy INTERFACE cxx_auto_type)
+target_compile_features(vtkm_diy INTERFACE cxx_auto_type)
 
 # placeholder to support external DIY
 set(VTKM_USE_EXTERNAL_DIY OFF)
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Configure.h.in
   ${VTKm_BINARY_INCLUDE_DIR}/${kit_dir}/Configure.h)
 
-target_include_directories(diy INTERFACE
+target_include_directories(vtkm_diy INTERFACE
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
   $<INSTALL_INTERFACE:${VTKm_INSTALL_INCLUDE_DIR}>)
 
-# presently, this dependency is required. Make it optional in the future.
+if(VTKm_ENABLE_MPI)
-set(arg)
+  set(arg)
-foreach(apath IN LISTS MPI_C_INCLUDE_PATH MPI_CXX_INCLUDE_PATH)
+  foreach(apath IN LISTS MPI_C_INCLUDE_PATH MPI_CXX_INCLUDE_PATH)
-  list(APPEND arg $<BUILD_INTERFACE:${apath}>)
+    list(APPEND arg $<BUILD_INTERFACE:${apath}>)
-endforeach()
+  endforeach()
-target_include_directories(diy INTERFACE ${arg})
+  list(REMOVE_DUPLICATES arg)
-
+  target_include_directories(vtkm_diy INTERFACE ${arg})
-target_link_libraries(diy INTERFACE
+  target_link_libraries(vtkm_diy INTERFACE
-  $<BUILD_INTERFACE:${MPI_C_LIBRARIES}>
+    $<BUILD_INTERFACE:${MPI_C_LIBRARIES}>
-  $<BUILD_INTERFACE:${MPI_CXX_LIBRARIES}>)
+    $<BUILD_INTERFACE:${MPI_CXX_LIBRARIES}>)
-
+  if(MPI_C_COMPILE_DEFINITIONS)
-if(MPI_C_COMPILE_DEFINITIONS)
+    target_compile_definitions(vtkm_diy INTERFACE
-  target_compile_definitions(diy INTERFACE
+      $<$<COMPILE_LANGUAGE:C>:${MPI_C_COMPILE_DEFINITIONS}>)
-    $<$<COMPILE_LANGUAGE:C>:${MPI_C_COMPILE_DEFINITIONS}>)
+  endif()
-endif()
+  if(MPI_CXX_COMPILE_DEFNITIONS)
-if(MPI_CXX_COMPILE_DEFNITIONS)
+    target_compile_definitions(vtkm_diy INTERFACE
-  target_compile_definitions(diy INTERFACE
+      $<$<COMPILE_LANGUAGE:CXX>:${MPI_CXX_COMPILE_DEFNITIONS>)
-    $<$<COMPILE_LANGUAGE:CXX>:${MPI_CXX_COMPILE_DEFNITIONS>)
+  endif()
 endif()
 
-install(TARGETS diy
+install(TARGETS vtkm_diy
   EXPORT ${VTKm_EXPORT_NAME})
 
 ## Install headers

vtkm/thirdparty/diy/Configure.h.in

@@ -22,9 +22,21 @@
 #ifndef vtkm_diy_h
 #define vtkm_diy_h
 
+#include <vtkm/internal/Configure.h>
+
 /* Use the diy library configured for VTM-m. */
 #cmakedefine01 VTKM_USE_EXTERNAL_DIY
 
+/* Whether to use MPI support in DIY */
+#if !defined(VTKM_ENABLE_MPI)
+# define DIY_NO_MPI
+#endif
+
+/* initially, we disable DIY threads.
+ * once we've sorted out how DIY threads and vtkm work together
+ * we will make this configurable.*/
+#define DIY_NO_THREADS
+
 #if VTKM_USE_EXTERNAL_DIY
 # define VTKM_DIY(header) <header>
 #else