Started adding DIY code for HierarchicalAugmenter

examples/contour_tree_distributed/ContourTreeApp.cxx

@@ -840,6 +840,7 @@ int main(int argc, char* argv[])
                                                      localBlockSizes,
                                                      useBoundaryExtremaOnly,
                                                      useMarchingCubes,
+                                                     false,
                                                      saveDotFiles,
                                                      timingsLogLevel,
                                                      treeLogLevel);

vtkm/filter/ContourTreeUniformDistributed.h

@@ -118,6 +118,7 @@ public:
     const vtkm::cont::ArrayHandle<vtkm::Id3>& localBlockSizes,
     bool useBoundaryExtremaOnly = true,
     bool useMarchingCubes = false,
+    bool augmentHierarchicalTree = false,
     bool saveDotFiles = false,
     vtkm::cont::LogLevel timingsLogLevel = vtkm::cont::LogLevel::Perf,
     vtkm::cont::LogLevel treeLogLevel = vtkm::cont::LogLevel::Info);
@@ -183,6 +184,9 @@ private:
   /// Use marching cubes connectivity for computing the contour tree
   bool UseMarchingCubes;
 
+  /// Augment hierarchical tree
+  bool AugmentHierarchicalTree;
+
   /// Save dot files for all tree computations
   bool SaveDotFiles;
 

vtkm/filter/ContourTreeUniformDistributed.hxx

@@ -68,7 +68,6 @@
 #include <vtkm/worklet/contourtree_distributed/BoundaryTreeMaker.h>
 #include <vtkm/worklet/contourtree_distributed/ComputeDistributedContourTreeFunctor.h>
 #include <vtkm/worklet/contourtree_distributed/DistributedContourTreeBlockData.h>
-#include <vtkm/worklet/contourtree_distributed/HierarchicalContourTree.h>
 #include <vtkm/worklet/contourtree_distributed/InteriorForest.h>
 #include <vtkm/worklet/contourtree_distributed/PrintGraph.h>
 #include <vtkm/worklet/contourtree_distributed/SpatialDecomposition.h>
@@ -169,8 +168,8 @@ void SaveAfterFanInResults(
 
 template <typename FieldType>
 void SaveHierarchicalTreeDot(
-  vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>* blockData,
+  const vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>*
-  vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType>& hierarchicalTree,
+    blockData,
   vtkm::Id rank,
   vtkm::Id nRounds)
 {
@@ -190,7 +189,7 @@ void SaveHierarchicalTreeDot(
   std::ofstream hierarchicalTreeFile(hierarchicalTreeFileName);
   hierarchicalTreeFile
     << vtkm::worklet::contourtree_distributed::HierarchicalContourTreeDotGraphPrint<FieldType>(
-         hierarchicalTreeLabel, hierarchicalTree, hierarchicalTreeDotSettings);
+         hierarchicalTreeLabel, blockData->HierarchicalTree, hierarchicalTreeDotSettings);
 }
 
 } // end namespace contourtree_distributed_detail
@@ -207,12 +206,14 @@ ContourTreeUniformDistributed::ContourTreeUniformDistributed(
   const vtkm::cont::ArrayHandle<vtkm::Id3>& localBlockSizes,
   bool useBoundaryExtremaOnly,
   bool useMarchingCubes,
+  bool augmentHierarchicalTree,
   bool saveDotFiles,
   vtkm::cont::LogLevel timingsLogLevel,
   vtkm::cont::LogLevel treeLogLevel)
   : vtkm::filter::FilterField<ContourTreeUniformDistributed>()
   , UseBoundaryExtremaOnly(useBoundaryExtremaOnly)
   , UseMarchingCubes(useMarchingCubes)
+  , AugmentHierarchicalTree(augmentHierarchicalTree)
   , SaveDotFiles(saveDotFiles)
   , TimingsLogLevel(timingsLogLevel)
   , TreeLogLevel(treeLogLevel)
@@ -863,9 +864,6 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
   timer.Start();
 
   // 2. Fan out to update all the tree
-  // 2.1  DataSets for creating output data
-  std::vector<vtkm::cont::DataSet> hierarchicalTreeOutputDataSet(localDataBlocks.size());
-  // 2.2. Use foreach to compute the fan-out
   master.foreach (
     [&](
       vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>* blockData,
@@ -882,8 +880,7 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
       // Fan out
       auto nRounds = blockData->ContourTrees.size() - 1;
 
-      vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType> hierarchicalTree;
+      blockData->HierarchicalTree.Initialize(static_cast<vtkm::Id>(nRounds),
-      hierarchicalTree.Initialize(static_cast<vtkm::Id>(nRounds),
                                              blockData->ContourTrees[nRounds],
                                              blockData->ContourTreeMeshes[nRounds - 1]);
 
@@ -891,7 +888,7 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
       if (this->SaveDotFiles)
       {
         vtkm::filter::contourtree_distributed_detail::SaveHierarchicalTreeDot(
-          blockData, hierarchicalTree, rank, nRounds);
+          blockData, rank, nRounds);
       } // if(this->SaveDotFiles)
 
       fanoutTimingsStream << "    Fan Out Init Hierarchical Tree (block=" << blockData->BlockIndex
@@ -907,13 +904,13 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
                     blockData->ContourTrees[round],
                     &(blockData->InteriorForests[round]));
         grafter.GraftInteriorForests(static_cast<vtkm::Id>(round),
-                                     hierarchicalTree,
+                                     blockData->HierarchicalTree,
                                      blockData->ContourTreeMeshes[round - 1].SortedValues);
         // save the corresponding .gv file
         if (this->SaveDotFiles)
         {
           vtkm::filter::contourtree_distributed_detail::SaveHierarchicalTreeDot(
-            blockData, hierarchicalTree, rank, nRounds);
+            blockData, rank, nRounds);
         } // if(this->SaveDotFiles)
         // Log the time for each of the iterations of the fan out loop
         fanoutTimingsStream << "    Fan Out Time (block=" << blockData->BlockIndex
@@ -939,29 +936,71 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
         this->MultiBlockSpatialDecomposition.LocalBlockSizes.ReadPortal().Get(
           blockData->BlockIndex),
         this->MultiBlockSpatialDecomposition.GlobalSize);
-      grafter.GraftInteriorForests(0, hierarchicalTree, fieldData, &localToGlobalIdRelabeler);
+      grafter.GraftInteriorForests(
+        0, blockData->HierarchicalTree, fieldData, &localToGlobalIdRelabeler);
 
       // Log the time for each of the iterations of the fan out loop
       fanoutTimingsStream << "    Fan Out Time (block=" << blockData->BlockIndex << " , round=" << 0
                           << ") : " << iterationTimer.GetElapsedTime() << " seconds" << std::endl;
+
+      // Log the timing stats we collected
+      VTKM_LOG_S(this->TimingsLogLevel,
+                 std::endl
+                   << "    ------------ Fan Out (block=" << blockData->BlockIndex
+                   << ")  ------------" << std::endl
+                   << fanoutTimingsStream.str());
+    });
+
+  if (this->AugmentHierarchicalTree)
+  {
+    master.foreach (
+      [&](vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>*
+            blockData,
+          const vtkmdiy::Master::ProxyWithLink&) {
+        blockData->HierarchicalAugmenter.Initialize(
+          blockData->BlockIndex, &blockData->HierarchicalTree, &blockData->AugmentedTree);
+      });
+
+    // TODO/FIXME: Exchange
+
+    master.foreach (
+      [&](vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>*
+            blockData,
+          const vtkmdiy::Master::ProxyWithLink&) {
+        blockData->HierarchicalAugmenter.BuildAugmentedTree();
+      });
+  }
+
+  // Create output data set
+  std::vector<vtkm::cont::DataSet> hierarchicalTreeOutputDataSet(localDataBlocks.size());
+  master.foreach (
+    [&](
+      vtkm::worklet::contourtree_distributed::DistributedContourTreeBlockData<FieldType>* blockData,
+      const vtkmdiy::Master::ProxyWithLink&) {
+      std::stringstream fanoutTimingsStream;
+      vtkm::cont::Timer iterationTimer;
       iterationTimer.Start();
 
       // Create data set from output
-      vtkm::cont::Field dataValuesField(
+      vtkm::cont::Field dataValuesField("DataValues",
-        "DataValues", vtkm::cont::Field::Association::WHOLE_MESH, hierarchicalTree.DataValues);
+                                        vtkm::cont::Field::Association::WHOLE_MESH,
+                                        blockData->HierarchicalTree.DataValues);
       hierarchicalTreeOutputDataSet[blockData->BlockIndex].AddField(dataValuesField);
       vtkm::cont::Field regularNodeGlobalIdsField("RegularNodeGlobalIds",
                                                   vtkm::cont::Field::Association::WHOLE_MESH,
-                                                  hierarchicalTree.RegularNodeGlobalIds);
+                                                  blockData->HierarchicalTree.RegularNodeGlobalIds);
       hierarchicalTreeOutputDataSet[blockData->BlockIndex].AddField(regularNodeGlobalIdsField);
-      vtkm::cont::Field superarcsField(
+      vtkm::cont::Field superarcsField("Superarcs",
-        "Superarcs", vtkm::cont::Field::Association::WHOLE_MESH, hierarchicalTree.Superarcs);
+                                       vtkm::cont::Field::Association::WHOLE_MESH,
+                                       blockData->HierarchicalTree.Superarcs);
       hierarchicalTreeOutputDataSet[blockData->BlockIndex].AddField(superarcsField);
-      vtkm::cont::Field supernodesField(
+      vtkm::cont::Field supernodesField("Supernodes",
-        "Supernodes", vtkm::cont::Field::Association::WHOLE_MESH, hierarchicalTree.Supernodes);
+                                        vtkm::cont::Field::Association::WHOLE_MESH,
+                                        blockData->HierarchicalTree.Supernodes);
       hierarchicalTreeOutputDataSet[blockData->BlockIndex].AddField(supernodesField);
-      vtkm::cont::Field superparentsField(
+      vtkm::cont::Field superparentsField("Superparents",
-        "Superparents", vtkm::cont::Field::Association::WHOLE_MESH, hierarchicalTree.Superparents);
+                                          vtkm::cont::Field::Association::WHOLE_MESH,
+                                          blockData->HierarchicalTree.Superparents);
       hierarchicalTreeOutputDataSet[blockData->BlockIndex].AddField(superparentsField);
 
       // Copy cell set from input data set. This is mainly to ensure that the output data set
@@ -978,8 +1017,9 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
       // save the corresponding .gv file
       if (this->SaveDotFiles)
       {
+        auto nRounds = blockData->ContourTrees.size() - 1;
         vtkm::filter::contourtree_distributed_detail::SaveHierarchicalTreeDot(
-          blockData, hierarchicalTree, rank, nRounds);
+          blockData, rank, nRounds);
 
         fanoutTimingsStream << "    Fan Out Save Dot (block=" << blockData->BlockIndex
                             << ") : " << iterationTimer.GetElapsedTime() << " seconds" << std::endl;
@@ -1000,7 +1040,7 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
                    << std::endl
                    << std::setw(42) << std::left << "    BlockIndex"
                    << ": " << blockData->BlockIndex << std::endl
-                   << hierarchicalTree.PrintTreeStats() << std::endl);
+                   << blockData->HierarchicalTree.PrintTreeStats() << std::endl);
     }); // master.foreach
 
   // Clean-up

vtkm/filter/testing/TestingContourTreeUniformDistributedFilter.h

@@ -293,6 +293,7 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed(
                                                      !useMarchingCubes,
                                                      useMarchingCubes,
                                                      false,
+                                                     false,
                                                      vtkm::cont::LogLevel::UserVerboseLast,
                                                      vtkm::cont::LogLevel::UserVerboseLast);
   filter.SetActiveField(fieldName);

vtkm/worklet/contourtree_distributed/DistributedContourTreeBlockData.h

@@ -56,6 +56,8 @@
 #include <vtkm/Types.h>
 #include <vtkm/worklet/contourtree_augmented/Types.h>
 #include <vtkm/worklet/contourtree_augmented/meshtypes/ContourTreeMesh.h>
+#include <vtkm/worklet/contourtree_distributed/HierarchicalAugmenter.h>
+#include <vtkm/worklet/contourtree_distributed/HierarchicalContourTree.h>
 
 // clang-format off
 VTKM_THIRDPARTY_PRE_INCLUDE
@@ -82,15 +84,22 @@ struct DistributedContourTreeBlockData
     delete static_cast<DistributedContourTreeBlockData<FieldType>*>(b);
   }
 
-  // Block data
+  // Block metadata
   vtkm::Id BlockIndex;
+  vtkm::Id3 BlockOrigin; // Origin of the data block
+  vtkm::Id3 BlockSize;   // Extends of the data block
+
+  // Fan in data
   std::vector<vtkm::worklet::contourtree_augmented::ContourTree> ContourTrees;
   std::vector<vtkm::worklet::contourtree_augmented::ContourTreeMesh<FieldType>> ContourTreeMeshes;
   std::vector<vtkm::worklet::contourtree_distributed::InteriorForest> InteriorForests;
 
-  // Block metadata
+  // Fan out data
-  vtkm::Id3 BlockOrigin; // Origin of the data block
+  vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType> HierarchicalTree;
-  vtkm::Id3 BlockSize;   // Extends of the data block
+
+  // Augmentation phase
+  vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType> HierarchicalAugmenter;
+  vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType> AugmentedTree;
 };
 } // namespace contourtree_distributed
 } // namespace worklet

vtkm/worklet/contourtree_distributed/HierarchicalAugmenter.h

@@ -259,7 +259,7 @@ void HierarchicalAugmenter<FieldType>::Initialize(
   {
     vtkm::worklet::contourtree_distributed::hierarchical_augmenter::IsAttachementPointPredicate
       isAttachementPointPredicate(
-        this->BaseTree->Superarcs, this->BaseTree->whichRound, this->BaseTree->NumRounds);
+        this->BaseTree->Superarcs, this->BaseTree->WhichRound, this->BaseTree->NumRounds);
     vtkm::cont::Algorithm::CopyIf(
       // first we generate a list of all of the supernodes
       vtkm::cont::ArrayHandleIndex(this->BaseTree->Supernodes.GetNumberOfValues()),
@@ -1311,4 +1311,39 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
 } // namespace worklet
 } // namespace vtkm
 
+namespace vtkmdiy
+{
+
+// Struct to serialize ContourTreeMesh objects (i.e., load/save) needed in parralle for DIY
+template <typename FieldType>
+struct Serialization<vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>>
+{
+  static void save(
+    vtkmdiy::BinaryBuffer& bb,
+    const vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>& ha)
+  {
+    vtkmdiy::save(bb, ha.OutGlobalRegularIds);
+    vtkmdiy::save(bb, ha.OutDataValues);
+    vtkmdiy::save(bb, ha.OutSupernodeIds);
+    vtkmdiy::save(bb, ha.OutSuperparents);
+    vtkmdiy::save(bb, ha.OutSuperparentRounds);
+    vtkmdiy::save(bb, ha.OutWhichRounds);
+  }
+
+  static void load(vtkmdiy::BinaryBuffer& bb,
+                   vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>& ha)
+  {
+    // TODO/FIXME: Save to Out or some other array?
+    vtkmdiy::load(bb, ha.OutGlobalRegularIds);
+    vtkmdiy::load(bb, ha.OutDataValues);
+    vtkmdiy::load(bb, ha.OutSupernodeIds);
+    vtkmdiy::load(bb, ha.OutSuperparents);
+    vtkmdiy::load(bb, ha.OutSuperparentRounds);
+    vtkmdiy::load(bb, ha.OutWhichRounds);
+  }
+};
+
+} // namespace mangled_vtkmdiy_namespace
+
+
 #endif

vtkm/worklet/contourtree_distributed/PrintGraph.h

@@ -1013,7 +1013,7 @@ template <typename FieldType>
 // template <typename FieldType, typename VectorType>
 std::string HierarchicalContourTreeDotGraphPrint(
   const std::string& label, // the label to use as title for the graph
-  vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType>&
+  const vtkm::worklet::contourtree_distributed::HierarchicalContourTree<FieldType>&
     hierarchicalTree, // the hierarchical contour tree itself
   const vtkm::Id showMask = vtkm::worklet::contourtree_distributed::
     SHOW_HIERARCHICAL_STANDARD) // mask with flags for what elements to show