From 934f085e09939916aaec85230b233007a664bb85 Mon Sep 17 00:00:00 2001 From: Sujin Philip Date: Mon, 8 Jun 2020 15:57:51 -0500 Subject: [PATCH] Build diy as a library Support both mpi and nompi versions simultaneously. --- .gitlab-ci.yml | 1 + .gitlab/ci/windows10.yml | 1 + CMake/VTKmConfig.cmake.in | 4 + CMake/VTKmDIYUtils.cmake | 59 ++++++ CMake/testing/VTKmCheckSourceInInstall.cmake | 1 + CMake/testing/VTKmTestWrappers.cmake | 185 ++++++++++++------ CMakeLists.txt | 1 + examples/CMakeLists.txt | 2 +- .../contour_tree_augmented/CMakeLists.txt | 2 +- .../contour_tree_augmented/ContourTreeApp.cxx | 2 +- examples/histogram/CMakeLists.txt | 2 +- examples/histogram/Histogram.cxx | 13 +- examples/histogram/HistogramMPI.hxx | 5 +- .../RedistributePoints.cxx | 2 +- .../redistribute_points/RedistributePoints.h | 4 +- vtkm/cont/EnvironmentTracker.cxx | 26 +-- vtkm/cont/testing/Testing.h | 27 +-- vtkm/filter/ContourTreeUniformAugmented.hxx | 6 +- .../testing/UnitTestHistogramFilter.cxx | 5 + ...tTestPartitionedDataSetHistogramFilter.cxx | 5 + vtkm/thirdparty/diy/CMakeLists.txt | 106 ++++++++-- vtkm/thirdparty/diy/Configure.h.in | 11 -- vtkm/thirdparty/diy/diy.h | 40 +--- vtkm/thirdparty/diy/environment.h | 19 ++ vtkm/thirdparty/diy/mpi-cast.h | 19 ++ vtkm/thirdparty/diy/post-include.h | 24 +++ vtkm/thirdparty/diy/pre-include.h | 15 ++ vtkm/thirdparty/diy/serialization.h | 41 +--- 28 files changed, 418 insertions(+), 210 deletions(-) create mode 100644 CMake/VTKmDIYUtils.cmake create mode 100644 vtkm/thirdparty/diy/environment.h create mode 100644 vtkm/thirdparty/diy/mpi-cast.h create mode 100644 vtkm/thirdparty/diy/post-include.h create mode 100644 vtkm/thirdparty/diy/pre-include.h diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa569bd24..da53aa973 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,6 +125,7 @@ stages: # The artifacts of the build. - build/bin/ - build/include/ + - build/vtkm/thirdparty/diy/vtkmdiy/ - build/lib/ # CTest and CMake install files. diff --git a/.gitlab/ci/windows10.yml b/.gitlab/ci/windows10.yml index 464f89acf..3cdb3efd8 100644 --- a/.gitlab/ci/windows10.yml +++ b/.gitlab/ci/windows10.yml @@ -18,6 +18,7 @@ - build/bin/ - build/include/ - build/lib/ + - build/vtkm/thirdparty/diy/vtkmdiy/include # CTest and CMake install files. # XXX(globbing): Can be simplified with support from diff --git a/CMake/VTKmConfig.cmake.in b/CMake/VTKmConfig.cmake.in index d2d251a43..24eecedfb 100644 --- a/CMake/VTKmConfig.cmake.in +++ b/CMake/VTKmConfig.cmake.in @@ -116,3 +116,7 @@ endif() # This includes a host of functions used by VTK-m CMake. include(VTKmWrappers) include(VTKmRenderingContexts) + +# Setup diy magic of chosing the appropriate mpi/no_mpi library to link against +include(VTKmDIYUtils) +vtkm_diy_init_target() diff --git a/CMake/VTKmDIYUtils.cmake b/CMake/VTKmDIYUtils.cmake new file mode 100644 index 000000000..1594ba1fe --- /dev/null +++ b/CMake/VTKmDIYUtils.cmake @@ -0,0 +1,59 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +macro(_vtkm_diy_target flag target) + set(${target} "vtkmdiympi") + if (NOT ${flag}) + set(${target} "vtkmdiympi_nompi") + endif() +endmacro() + +function(vtkm_diy_init_target) + set(vtkm_diy_default_flag "${VTKm_ENABLE_MPI}") + _vtkm_diy_target(vtkm_diy_default_flag vtkm_diy_default_target) + + set_target_properties(vtkm_diy PROPERTIES + vtkm_diy_use_mpi_stack ${vtkm_diy_default_flag} + vtkm_diy_target ${vtkm_diy_default_target}) +endfunction() + +#----------------------------------------------------------------------------- +function(vtkm_diy_use_mpi_push) + set(topval ${VTKm_ENABLE_MPI}) + if (NOT ARGC EQUAL 0) + set(topval ${ARGV0}) + endif() + get_target_property(stack vtkm_diy vtkm_diy_use_mpi_stack) + list (APPEND stack ${topval}) + _vtkm_diy_target(topval target) + set_target_properties(vtkm_diy PROPERTIES + vtkm_diy_use_mpi_stack "${stack}" + vtkm_diy_target "${target}") +endfunction() + +function(vtkm_diy_use_mpi value) + get_target_property(stack vtkm_diy vtkm_diy_use_mpi_stack) + list (REMOVE_AT stack -1) + list (APPEND stack ${value}) + _vtkm_diy_target(value target) + set_target_properties(vtkm_diy PROPERTIES + vtkm_diy_use_mpi_stack "${stack}" + vtkm_diy_target "${target}") +endfunction() + +function(vtkm_diy_use_mpi_pop) + get_target_property(stack vtkm_diy vtkm_diy_use_mpi_stack) + list (GET stack -1 value) + list (REMOVE_AT stack -1) + _vtkm_diy_target(value target) + set_target_properties(vtkm_diy PROPERTIES + vtkm_diy_use_mpi_stack "${stack}" + vtkm_diy_target "${target}") +endfunction() diff --git a/CMake/testing/VTKmCheckSourceInInstall.cmake b/CMake/testing/VTKmCheckSourceInInstall.cmake index ebe550df3..fd0d771ca 100644 --- a/CMake/testing/VTKmCheckSourceInInstall.cmake +++ b/CMake/testing/VTKmCheckSourceInInstall.cmake @@ -111,6 +111,7 @@ function(do_verify root_dir prefix) set(file_exceptions cont/ColorTablePrivate.hxx + thirdparty/diy/vtkmdiy/cmake/mpi_types.h ) #by default every header in a testing directory doesn't need to be installed diff --git a/CMake/testing/VTKmTestWrappers.cmake b/CMake/testing/VTKmTestWrappers.cmake index 8ec6042f9..b97ce7101 100644 --- a/CMake/testing/VTKmTestWrappers.cmake +++ b/CMake/testing/VTKmTestWrappers.cmake @@ -10,6 +10,69 @@ include(VTKmWrappers) +function(vtkm_create_test_executable + prog_name + sources + libraries + defines + is_mpi_test + use_mpi + enable_all_backends + use_job_pool) + + vtkm_diy_use_mpi_push() + + set(prog ${prog_name}) + + # for MPI tests, suffix test name and add MPI_Init/MPI_Finalize calls. + if (is_mpi_test) + set(extraArgs EXTRA_INCLUDE "vtkm/thirdparty/diy/environment.h") + set(CMAKE_TESTDRIVER_BEFORE_TESTMAIN "vtkmdiy::mpi::environment env(ac, av);") + + if (use_mpi) + vtkm_diy_use_mpi(ON) + set(prog "${prog}_mpi") + else() + vtkm_diy_use_mpi(OFF) + set(prog "${prog}_nompi") + endif() + else() + set(CMAKE_TESTDRIVER_BEFORE_TESTMAIN "") + endif() + + #the creation of the test source list needs to occur before the labeling as + #cuda. This is so that we get the correctly named entry points generated + create_test_sourcelist(test_sources ${prog}.cxx ${sources} ${extraArgs}) + + add_executable(${prog} ${prog}.cxx ${sources}) + vtkm_add_drop_unused_function_flags(${prog}) + target_compile_definitions(${prog} PRIVATE ${defines}) + + #if all backends are enabled, we can use cuda compiler to handle all possible backends. + set(device_sources) + if(TARGET vtkm::cuda AND enable_all_backends) + set(device_sources ${sources}) + endif() + vtkm_add_target_information(${prog} DEVICE_SOURCES ${device_sources}) + + if(NOT VTKm_USE_DEFAULT_SYMBOL_VISIBILITY) + set_property(TARGET ${prog} PROPERTY CUDA_VISIBILITY_PRESET "hidden") + set_property(TARGET ${prog} PROPERTY CXX_VISIBILITY_PRESET "hidden") + endif() + set_property(TARGET ${prog} PROPERTY ARCHIVE_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) + set_property(TARGET ${prog} PROPERTY LIBRARY_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) + set_property(TARGET ${prog} PROPERTY RUNTIME_OUTPUT_DIRECTORY ${VTKm_EXECUTABLE_OUTPUT_PATH}) + + target_link_libraries(${prog} PRIVATE vtkm_cont ${libraries}) + + if(use_job_pool) + vtkm_setup_job_pool() + set_property(TARGET ${prog} PROPERTY JOB_POOL_COMPILE vtkm_pool) + endif() + + vtkm_diy_use_mpi_pop() +endfunction() + #----------------------------------------------------------------------------- # Declare unit tests, which should be in the same directory as a kit # (package, module, whatever you call it). Usage: @@ -36,7 +99,9 @@ include(VTKmWrappers) # test executable # # [MPI] : when specified, the tests should be run in parallel if -# MPI is enabled. +# MPI is enabled. The tests should also be able to build and run +# When MPI is not available, i.e., they should not make explicit +# use of MPI and instead completely rely on DIY. # [ALL_BACKENDS] : when specified, the tests would test against all enabled # backends. Otherwise we expect the tests to manage the # backends at runtime. @@ -56,9 +121,6 @@ function(vtkm_unit_tests) ) vtkm_parse_test_options(VTKm_UT_SOURCES "${options}" ${VTKm_UT_SOURCES}) - set(test_prog) - - set(per_device_command_line_arguments "NONE") set(per_device_suffix "") set(per_device_timeout 180) @@ -93,6 +155,7 @@ function(vtkm_unit_tests) endif() endif() + set(test_prog) if(VTKm_UT_NAME) set(test_prog "${VTKm_UT_NAME}") else() @@ -110,43 +173,38 @@ function(vtkm_unit_tests) list(APPEND VTKm_UT_TEST_ARGS "--baseline-dir=${VTKm_SOURCE_DIR}/data/baseline") if(VTKm_UT_MPI) - # for MPI tests, suffix test name and add MPI_Init/MPI_Finalize calls. - set(test_prog "${test_prog}_mpi") - set(extraArgs EXTRA_INCLUDE "vtkm/cont/testing/Testing.h" - FUNCTION "vtkm::cont::testing::Environment env") + if (VTKm_ENABLE_MPI) + vtkm_create_test_executable( + ${test_prog} + "${VTKm_UT_SOURCES}" + "${VTKm_UT_LIBRARIES}" + "${VTKm_UT_DEFINES}" + ON # is_mpi_test + ON # use_mpi + ${enable_all_backends} + ${VTKm_UT_USE_VTKM_JOB_POOL}) + endif() + if ((NOT VTKm_ENABLE_MPI) OR VTKm_ENABLE_DIY_NOMPI) + vtkm_create_test_executable( + ${test_prog} + "${VTKm_UT_SOURCES}" + "${VTKm_UT_LIBRARIES}" + "${VTKm_UT_DEFINES}" + ON # is_mpi_test + OFF # use_mpi + ${enable_all_backends} + ${VTKm_UT_USE_VTKM_JOB_POOL}) + endif() else() - set(extraArgs) - endif() - - #the creation of the test source list needs to occur before the labeling as - #cuda. This is so that we get the correctly named entry points generated - create_test_sourcelist(test_sources ${test_prog}.cxx ${VTKm_UT_SOURCES} ${extraArgs}) - - add_executable(${test_prog} ${test_prog}.cxx ${VTKm_UT_SOURCES}) - vtkm_add_drop_unused_function_flags(${test_prog}) - target_compile_definitions(${test_prog} PRIVATE ${VTKm_UT_DEFINES}) - - - #if all backends are enabled, we can use cuda compiler to handle all possible backends. - set(device_sources ) - if(TARGET vtkm::cuda AND enable_all_backends) - set(device_sources ${VTKm_UT_SOURCES}) - endif() - vtkm_add_target_information(${test_prog} DEVICE_SOURCES ${device_sources}) - - if(VTKm_HIDE_PRIVATE_SYMBOLS) - set_property(TARGET ${test_prog} PROPERTY CUDA_VISIBILITY_PRESET "hidden") - set_property(TARGET ${test_prog} PROPERTY CXX_VISIBILITY_PRESET "hidden") - endif() - set_property(TARGET ${test_prog} PROPERTY ARCHIVE_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) - set_property(TARGET ${test_prog} PROPERTY LIBRARY_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) - set_property(TARGET ${test_prog} PROPERTY RUNTIME_OUTPUT_DIRECTORY ${VTKm_EXECUTABLE_OUTPUT_PATH}) - - target_link_libraries(${test_prog} PRIVATE vtkm_cont ${VTKm_UT_LIBRARIES}) - - if(VTKm_UT_USE_VTKM_JOB_POOL) - vtkm_setup_job_pool() - set_property(TARGET ${test_prog} PROPERTY JOB_POOL_COMPILE vtkm_pool) + vtkm_create_test_executable( + ${test_prog} + "${VTKm_UT_SOURCES}" + "${VTKm_UT_LIBRARIES}" + "${VTKm_UT_DEFINES}" + OFF # is_mpi_test + OFF # use_mpi + ${enable_all_backends} + ${VTKm_UT_USE_VTKM_JOB_POOL}) endif() list(LENGTH per_device_command_line_arguments number_of_devices) @@ -170,25 +228,42 @@ function(vtkm_unit_tests) foreach (test ${VTKm_UT_SOURCES}) get_filename_component(tname ${test} NAME_WE) - if(VTKm_UT_MPI AND VTKm_ENABLE_MPI) - add_test(NAME ${tname}${upper_backend} - COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} - $ ${tname} ${device_command_line_argument} - ${vtkm_default_test_log_level} ${VTKm_UT_TEST_ARGS} ${MPIEXEC_POSTFLAGS} - ) - else() + if(VTKm_UT_MPI) + if (VTKm_ENABLE_MPI) + add_test(NAME ${tname}${upper_backend}_mpi + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} + $ ${tname} ${device_command_line_argument} + ${vtkm_default_test_log_level} ${VTKm_UT_TEST_ARGS} ${MPIEXEC_POSTFLAGS} + ) + set_tests_properties("${tname}${upper_backend}_mpi" PROPERTIES + LABELS "${upper_backend};${VTKm_UT_LABEL}" + TIMEOUT ${timeout} + RUN_SERIAL ${run_serial} + FAIL_REGULAR_EXPRESSION "runtime error") + endif() # VTKm_ENABLE_MPI + if ((NOT VTKm_ENABLE_MPI) OR VTKm_ENABLE_DIY_NOMPI) + add_test(NAME ${tname}${upper_backend}_nompi + COMMAND ${test_prog}_nompi ${tname} ${device_command_line_argument} + ${vtkm_default_test_log_level} ${VTKm_UT_TEST_ARGS} + ) + set_tests_properties("${tname}${upper_backend}_nompi" PROPERTIES + LABELS "${upper_backend};${VTKm_UT_LABEL}" + TIMEOUT ${timeout} + RUN_SERIAL ${run_serial} + FAIL_REGULAR_EXPRESSION "runtime error") + + endif() # VTKm_ENABLE_DIY_NOMPI + else() # VTKm_UT_MPI add_test(NAME ${tname}${upper_backend} COMMAND ${test_prog} ${tname} ${device_command_line_argument} ${vtkm_default_test_log_level} ${VTKm_UT_TEST_ARGS} ) - endif() - - set_tests_properties("${tname}${upper_backend}" PROPERTIES - LABELS "${upper_backend};${VTKm_UT_LABEL}" - TIMEOUT ${timeout} - RUN_SERIAL ${run_serial} - FAIL_REGULAR_EXPRESSION "runtime error" - ) + set_tests_properties("${tname}${upper_backend}" PROPERTIES + LABELS "${upper_backend};${VTKm_UT_LABEL}" + TIMEOUT ${timeout} + RUN_SERIAL ${run_serial} + FAIL_REGULAR_EXPRESSION "runtime error") + endif() # VTKm_UT_MPI endforeach() endforeach() diff --git a/CMakeLists.txt b/CMakeLists.txt index ad9fde942..4084b096b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,6 +277,7 @@ if(NOT VTKm_INSTALL_ONLY_LIBRARIES) ${VTKm_SOURCE_DIR}/CMake/VTKmCPUVectorization.cmake ${VTKm_SOURCE_DIR}/CMake/VTKmDetectCUDAVersion.cu ${VTKm_SOURCE_DIR}/CMake/VTKmDeviceAdapters.cmake + ${VTKm_SOURCE_DIR}/CMake/VTKmDIYUtils.cmake ${VTKm_SOURCE_DIR}/CMake/VTKmExportHeaderTemplate.h.in ${VTKm_SOURCE_DIR}/CMake/VTKmMPI.cmake ${VTKm_SOURCE_DIR}/CMake/VTKmRenderingContexts.cmake diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index fdd50a678..4ec62e7fb 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -18,7 +18,7 @@ if(VTKm_ENABLE_EXAMPLES) add_subdirectory(contour_tree_augmented) add_subdirectory(cosmotools) add_subdirectory(demo) - add_subdirectory(game_of_life) + #add_subdirectory(game_of_life) add_subdirectory(hello_worklet) add_subdirectory(histogram) add_subdirectory(lagrangian) diff --git a/examples/contour_tree_augmented/CMakeLists.txt b/examples/contour_tree_augmented/CMakeLists.txt index 35d6be675..f008cca52 100644 --- a/examples/contour_tree_augmented/CMakeLists.txt +++ b/examples/contour_tree_augmented/CMakeLists.txt @@ -78,7 +78,7 @@ endif() #################################### if (VTKm_ENABLE_MPI) add_executable(ContourTree_Augmented_MPI ContourTreeApp.cxx) - target_link_libraries(ContourTree_Augmented_MPI vtkm_filter vtkm_io) + target_link_libraries(ContourTree_Augmented_MPI vtkm_filter vtkm_io MPI::MPI_CXX) vtkm_add_target_information(ContourTree_Augmented_MPI MODIFY_CUDA_FLAGS DEVICE_SOURCES ContourTreeApp.cxx) diff --git a/examples/contour_tree_augmented/ContourTreeApp.cxx b/examples/contour_tree_augmented/ContourTreeApp.cxx index 7efffd5a6..310a91566 100644 --- a/examples/contour_tree_augmented/ContourTreeApp.cxx +++ b/examples/contour_tree_augmented/ContourTreeApp.cxx @@ -168,7 +168,7 @@ int main(int argc, char* argv[]) auto comm = MPI_COMM_WORLD; // Tell VTK-m which communicator it should use. - vtkm::cont::EnvironmentTracker::SetCommunicator(vtkmdiy::mpi::communicator(comm)); + vtkm::cont::EnvironmentTracker::SetCommunicator(vtkmdiy::mpi::communicator()); // get the rank and size int rank, size; diff --git a/examples/histogram/CMakeLists.txt b/examples/histogram/CMakeLists.txt index 89440647e..e160afb7b 100644 --- a/examples/histogram/CMakeLists.txt +++ b/examples/histogram/CMakeLists.txt @@ -14,7 +14,7 @@ project(Histogram CXX) find_package(VTKm REQUIRED QUIET) if (VTKm_ENABLE_MPI) add_executable(Histogram Histogram.cxx HistogramMPI.h HistogramMPI.hxx) - target_link_libraries(Histogram PRIVATE vtkm_filter) + target_link_libraries(Histogram PRIVATE vtkm_filter MPI::MPI_CXX) vtkm_add_target_information(Histogram DROP_UNUSED_SYMBOLS MODIFY_CUDA_FLAGS DEVICE_SOURCES Histogram.cxx) diff --git a/examples/histogram/Histogram.cxx b/examples/histogram/Histogram.cxx index 47b6bfb9b..5b1049ca7 100644 --- a/examples/histogram/Histogram.cxx +++ b/examples/histogram/Histogram.cxx @@ -57,14 +57,16 @@ int main(int argc, char* argv[]) vtkm::cont::Initialize(argc, argv, opts); // setup MPI environment. - MPI_Init(&argc, &argv); + vtkmdiy::mpi::environment env(argc, argv); // will finalize on destruction + + vtkmdiy::mpi::communicator world; // the default is MPI_COMM_WORLD // tell VTK-m the communicator to use. - vtkm::cont::EnvironmentTracker::SetCommunicator(vtkmdiy::mpi::communicator(MPI_COMM_WORLD)); + vtkm::cont::EnvironmentTracker::SetCommunicator(world); int rank, size; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_rank(vtkmdiy::mpi::mpi_cast(world.handle()), &rank); + MPI_Comm_size(vtkmdiy::mpi::mpi_cast(world.handle()), &size); if (argc != 2) { @@ -72,7 +74,6 @@ int main(int argc, char* argv[]) { std::cout << "Usage: " << std::endl << "$ " << argv[0] << " " << std::endl; } - MPI_Finalize(); return EXIT_FAILURE; } @@ -105,11 +106,9 @@ int main(int argc, char* argv[]) if (count != numVals * size) { std::cout << "ERROR: bins mismatched!" << std::endl; - MPI_Finalize(); return EXIT_FAILURE; } } - MPI_Finalize(); return EXIT_SUCCESS; } diff --git a/examples/histogram/HistogramMPI.hxx b/examples/histogram/HistogramMPI.hxx index c6def13c2..c23bcee5e 100644 --- a/examples/histogram/HistogramMPI.hxx +++ b/examples/histogram/HistogramMPI.hxx @@ -18,6 +18,9 @@ #include #include +#include + +#include namespace example { @@ -81,7 +84,7 @@ public: sizeof(vtkm::Id) == 4 ? MPI_INT : MPI_LONG, MPI_SUM, 0, - comm); + vtkmdiy::mpi::mpi_cast(comm.handle())); if (comm.rank() == 0) { diff --git a/examples/redistribute_points/RedistributePoints.cxx b/examples/redistribute_points/RedistributePoints.cxx index efa8027ec..ec94f1de7 100644 --- a/examples/redistribute_points/RedistributePoints.cxx +++ b/examples/redistribute_points/RedistributePoints.cxx @@ -30,7 +30,7 @@ int main(int argc, char* argv[]) auto config = vtkm::cont::Initialize(argc, argv, opts); vtkmdiy::mpi::environment env(argc, argv); - auto comm = vtkmdiy::mpi::communicator(MPI_COMM_WORLD); + vtkmdiy::mpi::communicator comm; vtkm::cont::EnvironmentTracker::SetCommunicator(comm); if (argc != 3) diff --git a/examples/redistribute_points/RedistributePoints.h b/examples/redistribute_points/RedistributePoints.h index 447c4a1b2..4b055228f 100644 --- a/examples/redistribute_points/RedistributePoints.h +++ b/examples/redistribute_points/RedistributePoints.h @@ -27,7 +27,7 @@ namespace internal static vtkmdiy::ContinuousBounds convert(const vtkm::Bounds& bds) { - vtkmdiy::ContinuousBounds result; + vtkmdiy::ContinuousBounds result(3); result.min[0] = static_cast(bds.X.Min); result.min[1] = static_cast(bds.Y.Min); result.min[2] = static_cast(bds.Z.Min); @@ -136,7 +136,7 @@ public: { auto target = rp.out_link().target(cc); // let's get the bounding box for the target block. - vtkmdiy::ContinuousBounds bds; + vtkmdiy::ContinuousBounds bds(3); this->Decomposer.fill_bounds(bds, target.gid); auto extractedDS = this->Extract(*block, bds); diff --git a/vtkm/cont/EnvironmentTracker.cxx b/vtkm/cont/EnvironmentTracker.cxx index cb03cb736..924015889 100644 --- a/vtkm/cont/EnvironmentTracker.cxx +++ b/vtkm/cont/EnvironmentTracker.cxx @@ -11,34 +11,36 @@ #include +#include + namespace vtkm { namespace cont { namespace internal { -static vtkmdiy::mpi::communicator GlobalCommuncator(MPI_COMM_NULL); +static std::unique_ptr GlobalCommuncator; } void EnvironmentTracker::SetCommunicator(const vtkmdiy::mpi::communicator& comm) { - vtkm::cont::internal::GlobalCommuncator = comm; + if (!internal::GlobalCommuncator) + { + internal::GlobalCommuncator.reset(new vtkmdiy::mpi::communicator(comm)); + } + else + { + *internal::GlobalCommuncator = comm; + } } const vtkmdiy::mpi::communicator& EnvironmentTracker::GetCommunicator() { -#ifndef VTKM_DIY_NO_MPI - int flag; - MPI_Initialized(&flag); - if (!flag) + if (!internal::GlobalCommuncator) { - int argc = 0; - char** argv = nullptr; - MPI_Init(&argc, &argv); - internal::GlobalCommuncator = vtkmdiy::mpi::communicator(MPI_COMM_WORLD); + internal::GlobalCommuncator.reset(new vtkmdiy::mpi::communicator()); } -#endif - return vtkm::cont::internal::GlobalCommuncator; + return *internal::GlobalCommuncator; } } // namespace vtkm::cont } // namespace vtkm diff --git a/vtkm/cont/testing/Testing.h b/vtkm/cont/testing/Testing.h index 0be345992..770fb0ba3 100644 --- a/vtkm/cont/testing/Testing.h +++ b/vtkm/cont/testing/Testing.h @@ -24,7 +24,7 @@ #include #include -#include +#include namespace opt = vtkm::cont::internal::option; @@ -265,31 +265,6 @@ private: } }; -struct Environment -{ - VTKM_CONT Environment(int* argc, char*** argv) - { -#if defined(VTKM_ENABLE_MPI) - int provided_threading; - MPI_Init_thread(argc, argv, MPI_THREAD_FUNNELED, &provided_threading); - - // set the global communicator to use in VTKm. - vtkmdiy::mpi::communicator comm(MPI_COMM_WORLD); - vtkm::cont::EnvironmentTracker::SetCommunicator(comm); -#else - (void)argc; - (void)argv; -#endif - } - - VTKM_CONT ~Environment() - { -#if defined(VTKM_ENABLE_MPI) - MPI_Finalize(); -#endif - } -}; - //============================================================================ class TestEqualResult { diff --git a/vtkm/filter/ContourTreeUniformAugmented.hxx b/vtkm/filter/ContourTreeUniformAugmented.hxx index a707d44ae..e91c75bc7 100644 --- a/vtkm/filter/ContourTreeUniformAugmented.hxx +++ b/vtkm/filter/ContourTreeUniformAugmented.hxx @@ -202,8 +202,7 @@ public: { if (this->NumberOfDimensions() == 2) { - // may need to change back when porting ot later verison of VTKM/vtkmdiy - vtkmdiy::DiscreteBounds domain; //(2); + vtkmdiy::DiscreteBounds domain(2); domain.min[0] = domain.min[1] = 0; domain.max[0] = static_cast(this->GlobalSize[0]); domain.max[1] = static_cast(this->GlobalSize[1]); @@ -211,8 +210,7 @@ public: } else { - // may need to change back when porting to later version of VTMK/vtkmdiy - vtkmdiy::DiscreteBounds domain; //(3); + vtkmdiy::DiscreteBounds domain(3); domain.min[0] = domain.min[1] = domain.min[2] = 0; domain.max[0] = static_cast(this->GlobalSize[0]); domain.max[1] = static_cast(this->GlobalSize[1]); diff --git a/vtkm/filter/testing/UnitTestHistogramFilter.cxx b/vtkm/filter/testing/UnitTestHistogramFilter.cxx index e34e7772d..8b3e06f15 100644 --- a/vtkm/filter/testing/UnitTestHistogramFilter.cxx +++ b/vtkm/filter/testing/UnitTestHistogramFilter.cxx @@ -13,6 +13,8 @@ #include #include +#include + // // Make a simple 2D, 1000 point dataset populated with stat distributions // @@ -328,5 +330,8 @@ void TestHistogram() int UnitTestHistogramFilter(int argc, char* argv[]) { + // Setup MPI environment: This test is not intendent to be run in parallel + // but filter does make some DIY/MPI calls + vtkmdiy::mpi::environment env(argc, argv); return vtkm::cont::testing::Testing::Run(TestHistogram, argc, argv); } diff --git a/vtkm/filter/testing/UnitTestPartitionedDataSetHistogramFilter.cxx b/vtkm/filter/testing/UnitTestPartitionedDataSetHistogramFilter.cxx index 1e9445f3a..6ddd013dc 100644 --- a/vtkm/filter/testing/UnitTestPartitionedDataSetHistogramFilter.cxx +++ b/vtkm/filter/testing/UnitTestPartitionedDataSetHistogramFilter.cxx @@ -13,6 +13,8 @@ #include #include +#include + #include #include #include @@ -127,5 +129,8 @@ static void TestPartitionedDataSetHistogram() int UnitTestPartitionedDataSetHistogramFilter(int argc, char* argv[]) { + // Setup MPI environment: This test is not intendent to be run in parallel + // but filter does make some DIY/MPI calls + vtkmdiy::mpi::environment env(argc, argv); return vtkm::cont::testing::Testing::Run(TestPartitionedDataSetHistogram, argc, argv); } diff --git a/vtkm/thirdparty/diy/CMakeLists.txt b/vtkm/thirdparty/diy/CMakeLists.txt index af6303fe2..d6f892695 100644 --- a/vtkm/thirdparty/diy/CMakeLists.txt +++ b/vtkm/thirdparty/diy/CMakeLists.txt @@ -7,36 +7,118 @@ ## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ## PURPOSE. See the above copyright notice for more information. ##============================================================================ -add_library(vtkm_diy INTERFACE) - vtkm_get_kit_name(kit_name kit_dir) -# diy needs C++11 -target_compile_features(vtkm_diy INTERFACE cxx_std_11) +include(CMakeDependentOption) +if (NOT DEFINED VTKm_ENABLE_DIY_NOMPI) + cmake_dependent_option( + VTKm_ENABLE_DIY_NOMPI "Also build DIY without mpi" OFF "VTKm_ENABLE_MPI" OFF) +endif() + +if (VTKm_ENABLE_DIY_NOMPI AND + (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") AND + BUILD_SHARED_LIBS) + message(WARNING "VTKm_ENABLE_DIY_NOMPI support with MSVC shared builds is experimental and may not work.") +endif() + +## Configure diy +set(build_examples OFF) +set(build_tests OFF) + +set(threads OFF) +set(log OFF) +set(profile OFF) +set(caliper OFF) + +set(build_diy_mpi_lib ON) + +set(mpi OFF) +set(build_diy_nompi_lib OFF) +if (VTKm_ENABLE_MPI) + set(mpi ON) +endif() +if (VTKm_ENABLE_DIY_NOMPI) + set(build_diy_nompi_lib ON) +endif() + +mark_as_advanced(FORCE caliper log profile wrapped_mpi) + +set(diy_prefix "vtkmdiy") +set(diy_install_include_dir ${VTKm_INSTALL_INCLUDE_DIR}/${kit_dir}/vtkmdiy/include) +set(diy_install_lib_dir ${VTKm_INSTALL_LIB_DIR}) +set(diy_export_name ${VTKm_EXPORT_NAME}) +if (VTKm_INSTALL_ONLY_LIBRARIES) + set(diy_install_only_libraries) +endif() +set(diy_dont_install_export) # placeholder to support external DIY set(VTKM_USE_EXTERNAL_DIY OFF) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Configure.h.in ${VTKm_BINARY_INCLUDE_DIR}/${kit_dir}/Configure.h) +function(vtkm_diy_set_target_output_directory target) + set_property(TARGET ${target} PROPERTY ARCHIVE_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) + set_property(TARGET ${target} PROPERTY LIBRARY_OUTPUT_DIRECTORY ${VTKm_LIBRARY_OUTPUT_PATH}) + set_property(TARGET ${target} PROPERTY RUNTIME_OUTPUT_DIRECTORY ${VTKm_EXECUTABLE_OUTPUT_PATH}) +endfunction() + +#----------------------------------------------------------------------------- +add_subdirectory(vtkmdiy) + +# move diy libraries +if (TARGET vtkmdiympi) + vtkm_diy_set_target_output_directory(vtkmdiympi) +endif() +if (TARGET vtkmdiympi_nompi) + vtkm_diy_set_target_output_directory(vtkmdiympi_nompi) +endif() + +include(VTKmDIYUtils) + +add_library(vtkm_diy INTERFACE) +vtkm_diy_init_target() target_include_directories(vtkm_diy INTERFACE $ $) +target_link_libraries(vtkm_diy INTERFACE vtkmdiy) -if(VTKm_ENABLE_MPI) - target_link_libraries(vtkm_diy INTERFACE MPI::MPI_CXX) +# special logic for when both versions of the diy library are built +if (VTKm_ENABLE_DIY_NOMPI) + # only link vtkmdiympi/vtkmdiympi_nompi when building executable + set(is_exe "$,EXECUTABLE>") + target_link_libraries(vtkm_diy INTERFACE + "$>>") + + # ignore undefined symbols + set(is_shared_lib "$,SHARED_LIBRARY>") + if (APPLE) + target_link_libraries(vtkm_diy INTERFACE "$<${is_shared_lib}:-undefined dynamic_lookup>") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") + target_link_libraries(vtkm_diy INTERFACE + "$<${is_shared_lib}:-INCREMENTAL:NO>" + "$<${is_shared_lib}:-FORCE:UNRESOLVED>") + endif() +else() + if (TARGET vtkmdiympi) + target_link_libraries(vtkm_diy INTERFACE vtkmdiympi) + else() + target_link_libraries(vtkm_diy INTERFACE vtkmdiympi_nompi) + endif() endif() -install(TARGETS vtkm_diy - EXPORT ${VTKm_EXPORT_NAME}) +#----------------------------------------------------------------------------- +install(TARGETS vtkm_diy EXPORT ${VTKm_EXPORT_NAME}) - ## Install headers -if(NOT VTKm_INSTALL_ONLY_LIBRARIES) - install(DIRECTORY vtkmdiy - DESTINATION ${VTKm_INSTALL_INCLUDE_DIR}/${kit_dir}/) +## Install headers +if (NOT VTKm_INSTALL_ONLY_LIBRARIES) install(FILES ${VTKm_BINARY_INCLUDE_DIR}/${kit_dir}/Configure.h ${CMAKE_CURRENT_SOURCE_DIR}/diy.h + ${CMAKE_CURRENT_SOURCE_DIR}/environment.h + ${CMAKE_CURRENT_SOURCE_DIR}/mpi-cast.h + ${CMAKE_CURRENT_SOURCE_DIR}/post-include.h + ${CMAKE_CURRENT_SOURCE_DIR}/pre-include.h ${CMAKE_CURRENT_SOURCE_DIR}/serialization.h DESTINATION ${VTKm_INSTALL_INCLUDE_DIR}/${kit_dir}/) endif() diff --git a/vtkm/thirdparty/diy/Configure.h.in b/vtkm/thirdparty/diy/Configure.h.in index 94f898fe8..9365c39d5 100644 --- a/vtkm/thirdparty/diy/Configure.h.in +++ b/vtkm/thirdparty/diy/Configure.h.in @@ -20,16 +20,6 @@ /* Use the diy library configured for VTM-m. */ #cmakedefine01 VTKM_USE_EXTERNAL_DIY -/* Whether to use MPI support in DIY */ -#if !defined(VTKM_ENABLE_MPI) -# define VTKM_DIY_NO_MPI -#endif - -/* initially, we disable DIY threads. - * once we've sorted out how DIY threads and vtkm work together - * we will make this configurable.*/ -#define VTKM_DIY_NO_THREADS - /* Need to provide a way to for Serialziation * specializations to be injected into the correct * namespace. This solves the issue while allowing @@ -41,5 +31,4 @@ # define mangled_diy_namespace vtkmdiy #endif - #endif diff --git a/vtkm/thirdparty/diy/diy.h b/vtkm/thirdparty/diy/diy.h index a0737eb37..3ae41ae30 100644 --- a/vtkm/thirdparty/diy/diy.h +++ b/vtkm/thirdparty/diy/diy.h @@ -10,21 +10,9 @@ #ifndef vtk_m_thirdparty_diy_diy_h #define vtk_m_thirdparty_diy_diy_h -#include - -#if VTKM_USE_EXTERNAL_DIY -#define VTKM_DIY_INCLUDE(header) -#else -#define VTKM_DIY_INCLUDE(header) -#define diy vtkmdiy // mangle namespace diy (see below comments) -#endif - -#if defined(VTKM_CLANG) || defined(VTKM_GCC) -#pragma GCC visibility push(default) -#endif +#include "pre-include.h" // clang-format off -VTKM_THIRDPARTY_PRE_INCLUDE #include VTKM_DIY_INCLUDE(assigner.hpp) #include VTKM_DIY_INCLUDE(decomposition.hpp) #include VTKM_DIY_INCLUDE(master.hpp) @@ -36,29 +24,7 @@ VTKM_THIRDPARTY_PRE_INCLUDE #include VTKM_DIY_INCLUDE(reduce-operations.hpp) #include VTKM_DIY_INCLUDE(resolve.hpp) #include VTKM_DIY_INCLUDE(serialization.hpp) -#undef VTKM_DIY_INCLUDE -VTKM_THIRDPARTY_POST_INCLUDE // clang-format on +#include "post-include.h" -#if defined(VTKM_CLANG) || defined(VTKM_GCC) -#pragma GCC visibility pop -#endif - -// When using an external DIY -// We need to alias the diy namespace to -// vtkmdiy so that VTK-m uses it properly -#if VTKM_USE_EXTERNAL_DIY -namespace vtkmdiy = ::diy; - -#else -// The aliasing approach fails for when we -// want to us an internal version since -// the diy namespace already points to the -// external version. Instead we use macro -// replacement to make sure all diy classes -// are placed in vtkmdiy placed -#undef diy // mangle namespace diy - -#endif - -#endif +#endif // vtk_m_thirdparty_diy_diy_h diff --git a/vtkm/thirdparty/diy/environment.h b/vtkm/thirdparty/diy/environment.h new file mode 100644 index 000000000..2e5adfc2c --- /dev/null +++ b/vtkm/thirdparty/diy/environment.h @@ -0,0 +1,19 @@ +//============================================================================ +// Copyright (c) Kitware, Inc. +// All rights reserved. +// See LICENSE.txt for details. +// +// This software is distributed WITHOUT ANY WARRANTY; without even +// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. See the above copyright notice for more information. +//============================================================================ +#ifndef vtk_m_thirdparty_diy_environment_h +#define vtk_m_thirdparty_diy_environment_h + +#include "pre-include.h" +// clang-format off +#include VTKM_DIY_INCLUDE(mpi/environment.hpp) +// clang-format on +#include "post-include.h" + +#endif // vtk_m_thirdparty_diy_environment_h diff --git a/vtkm/thirdparty/diy/mpi-cast.h b/vtkm/thirdparty/diy/mpi-cast.h new file mode 100644 index 000000000..34786bc0c --- /dev/null +++ b/vtkm/thirdparty/diy/mpi-cast.h @@ -0,0 +1,19 @@ +//============================================================================ +// Copyright (c) Kitware, Inc. +// All rights reserved. +// See LICENSE.txt for details. +// +// This software is distributed WITHOUT ANY WARRANTY; without even +// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. See the above copyright notice for more information. +//============================================================================ +#ifndef vtk_m_thirdparty_diy_cast_h +#define vtk_m_thirdparty_diy_cast_h + +#include "pre-include.h" +// clang-format off +#include VTKM_DIY_INCLUDE(mpi/mpi_cast.hpp) +// clang-format on +#include "post-include.h" + +#endif // vtk_m_thirdparty_diy_cast_h diff --git a/vtkm/thirdparty/diy/post-include.h b/vtkm/thirdparty/diy/post-include.h new file mode 100644 index 000000000..e27037321 --- /dev/null +++ b/vtkm/thirdparty/diy/post-include.h @@ -0,0 +1,24 @@ +#undef VTKM_DIY_INCLUDE +VTKM_THIRDPARTY_POST_INCLUDE +// clang-format on + +#if defined(VTKM_CLANG) || defined(VTKM_GCC) +#pragma GCC visibility pop +#endif + +// When using an external DIY +// We need to alias the diy namespace to +// vtkmdiy so that VTK-m uses it properly +#if VTKM_USE_EXTERNAL_DIY +namespace vtkmdiy = ::diy; + +#else +// The aliasing approach fails for when we +// want to use an internal version since +// the diy namespace already points to the +// external version. Instead we use macro +// replacement to make sure all diy classes +// are placed in vtkmdiy placed +#undef diy // mangle namespace diy + +#endif diff --git a/vtkm/thirdparty/diy/pre-include.h b/vtkm/thirdparty/diy/pre-include.h new file mode 100644 index 000000000..b1753531a --- /dev/null +++ b/vtkm/thirdparty/diy/pre-include.h @@ -0,0 +1,15 @@ +#include + +#if VTKM_USE_EXTERNAL_DIY +#define VTKM_DIY_INCLUDE(header) +#else +#define VTKM_DIY_INCLUDE(header) +#define diy vtkmdiy // mangle namespace diy +#endif + +#if defined(VTKM_CLANG) || defined(VTKM_GCC) +#pragma GCC visibility push(default) +#endif + +// clang-format off +VTKM_THIRDPARTY_PRE_INCLUDE diff --git a/vtkm/thirdparty/diy/serialization.h b/vtkm/thirdparty/diy/serialization.h index a03e51ee2..128eb673b 100644 --- a/vtkm/thirdparty/diy/serialization.h +++ b/vtkm/thirdparty/diy/serialization.h @@ -10,45 +10,10 @@ #ifndef vtk_m_thirdparty_diy_serialization_h #define vtk_m_thirdparty_diy_serialization_h -#include - -#if VTKM_USE_EXTERNAL_DIY -#define VTKM_DIY_INCLUDE(header) -#else -#define VTKM_DIY_INCLUDE(header) -#define diy vtkmdiy // mangle namespace diy (see below comments) -#endif - -#if defined(VTKM_CLANG) || defined(VTKM_GCC) -#pragma GCC visibility push(default) -#endif - +#include "pre-include.h" // clang-format off -VTKM_THIRDPARTY_PRE_INCLUDE #include VTKM_DIY_INCLUDE(serialization.hpp) -#undef VTKM_DIY_INCLUDE -VTKM_THIRDPARTY_POST_INCLUDE // clang-format on +#include "post-include.h" -#if defined(VTKM_CLANG) || defined(VTKM_GCC) -#pragma GCC visibility pop -#endif - -// When using an external DIY -// We need to alias the diy namespace to -// vtkmdiy so that VTK-m uses it properly -#if VTKM_USE_EXTERNAL_DIY -namespace vtkmdiy = ::diy; - -#else -// The aliasing approach fails for when we -// want to us an internal version since -// the diy namespace already points to the -// external version. Instead we use macro -// replacement to make sure all diy classes -// are placed in vtkmdiy placed -#undef diy // mangle namespace diy - -#endif - -#endif +#endif // vtk_m_thirdparty_diy_serialization_h