Merge branch 'CreateSuperarcs-Debug' into 'add/presimplification'

Debug for presimplification

See merge request ghweber/vtk-m!3

examples/contour_tree_distributed/ContourTreeApp.cxx

@@ -938,7 +938,6 @@ int main(int argc, char* argv[])
     close(save_err);
   }
 
-  std::cout << "DONE!!!" << std::endl;
   // Finalize and finish the execution
   MPI_Finalize();
   return EXIT_SUCCESS;

vtkm/filter/scalar_topology/ContourTreeUniformDistributed.cxx

@@ -632,9 +632,17 @@ inline VTKM_CONT void ContourTreeUniformDistributed::ComputeVolumeMetric(
 
       // If we are pre-simplifying the tree then we need to use the base tree and if we compute the
       // final volume, then we need to use the augmented tree
+      /* ----------- MINGZHE: Bug fix here ---------  */
+      // currInBlock->HierarchicalAugmenter is NOT initialized
+      //     when this function is first called if pre-simplification is applied.
+      // currInBlock->HierarchicalAugmenter.AugmentedTree seems ok to remain,
+      //     because it is only called during augmentation,
+      //     in which the HierarchicalAugmenter is intialized.
       auto hierarchicalTreeToProcess = useAugmentedTree
         ? currInBlock->HierarchicalAugmenter.AugmentedTree
-        : currInBlock->HierarchicalAugmenter.BaseTree;
+        : &currInBlock->HierarchicalTree;
+
+      // Create HyperSweeper
       hierarchical_hyper_sweep_master.add(currInBlock->GlobalBlockId,
                                           new HyperSweepBlock(blockNo,
                                                               currInBlock->GlobalBlockId,
@@ -664,7 +672,7 @@ inline VTKM_CONT void ContourTreeUniformDistributed::ComputeVolumeMetric(
     localHypersweepTimer.Start();
 
     // Create HyperSweeper
-#ifdef DEBUG_PRINT
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
     VTKM_LOG_S(this->TreeLogLevel, "Block " << b->GlobalBlockId);
     VTKM_LOG_S(this->TreeLogLevel,
                b->HierarchicalContourTree.DebugPrint(
@@ -698,7 +706,7 @@ inline VTKM_CONT void ContourTreeUniformDistributed::ComputeVolumeMetric(
       hyperSweeper.InitializeIntrinsicVertexCount(
         b->HierarchicalContourTree, mesh, idRelabeler, b->IntrinsicVolume);
     }
-#ifdef DEBUG_PRINT
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
     VTKM_LOG_S(this->TreeLogLevel, "Block " << b->GlobalBlockId);
     VTKM_LOG_S(this->TreeLogLevel,
                b->HierarchicalContourTree.DebugPrint(
@@ -725,8 +733,8 @@ inline VTKM_CONT void ContourTreeUniformDistributed::ComputeVolumeMetric(
     // Perform the local hypersweep
     hyperSweeper.LocalHyperSweep();
 
-#ifdef DEBUG_PRINT
-    VTKM_LOG_S(this->Tree LogLevel, "Block " << b->GlobalBlockId);
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+    VTKM_LOG_S(this->TreeLogLevel, "Block " << b->GlobalBlockId);
     VTKM_LOG_S(this->TreeLogLevel,
                b->HierarchicalContourTree.DebugPrint("After local hypersweep", __FILE__, __LINE__));
 #endif
@@ -770,7 +778,7 @@ inline VTKM_CONT void ContourTreeUniformDistributed::ComputeVolumeMetric(
       intrinsicVolumes[b->LocalBlockNo] = b->IntrinsicVolume;
       dependentVolumes[b->LocalBlockNo] = b->DependentVolume;
 
-#ifdef DEBUG_PRINT
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
       VTKM_LOG_S(this->TreeLogLevel, "Block " << b->GlobalBlockId);
       VTKM_LOG_S(
         this->TreeLogLevel,
@@ -1201,6 +1209,7 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
                     partners,
                     vtkm::worklet::contourtree_distributed::HierarchicalAugmenterFunctor<FieldType>{
                       this->TimingsLogLevel });
+
     // Clear all swap data as it is no longer needed
     master.foreach (
       [](DistributedContourTreeBlockData* blockData, const vtkmdiy::Master::ProxyWithLink&) {

vtkm/filter/scalar_topology/worklet/contourtree_augmented/NotNoSuchElementPredicate.h

@@ -76,6 +76,19 @@ public:
 private:
 };
 
+//Simple functor to subset a VTKm ArrayHandle
+class NoSuchElementPredicate
+{
+public:
+  VTKM_EXEC_CONT
+  NoSuchElementPredicate() {}
+
+  VTKM_EXEC_CONT
+  bool operator()(const vtkm::Id& vertexId) const { return NoSuchElement(vertexId); }
+
+private:
+};
+
 } // namespace contourtree_augmented
 } // namespace worklet
 } // namespace vtkm

vtkm/filter/scalar_topology/worklet/contourtree_distributed/HierarchicalAugmenter.h

@@ -117,7 +117,6 @@
 #include <vtkm/filter/scalar_topology/worklet/contourtree_distributed/hierarchical_augmenter/UpdateHyperstructureSetSuperchildrenWorklet.h>
 #include <vtkm/filter/scalar_topology/worklet/contourtree_distributed/hierarchical_contour_tree/PermuteComparator.h>
 
-
 namespace vtkm
 {
 namespace worklet
@@ -277,10 +276,43 @@ void HierarchicalAugmenter<FieldType>::Initialize(
   this->MeshBlockSize = meshBockSize;
   this->MeshGlobalSize = meshGlobalSize;
 
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::stringstream headStr;
+  headStr << "=======================" << std::endl;
+  headStr << "Initializing Block " << blockId << std::endl;
+  headStr << "=======================" << std::endl;
+  VTKM_LOG_S(vtkm::cont::LogLevel::Info, headStr.str());
+#endif
+
   // now construct a list of all attachment points on the block
   // except those under the presimplify threshold. The presimplification is
   // handled in the IsAttachementPointPredicate
 
+  bool presimplify = ((volumeArray != NULL) && (presimplifyThreshold > 0));
+
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::stringstream presimpStr;
+  presimpStr << "Presimplification Threshold: " << presimplifyThreshold << std::endl;
+  presimpStr << "Volume Array:                " << ((volumeArray != NULL) ? "T" : "F") << std::endl;
+  presimpStr << "Threshold:                   " << ((presimplifyThreshold > 0) ? "T" : "F")
+             << std::endl;
+  presimpStr << "Presimplify: " << (presimplify ? "T" : "F") << std::endl;
+
+  if (presimplify)
+  { // presimplifying
+    vtkm::worklet::contourtree_augmented::PrintHeader((*volumeArray).GetNumberOfValues(),
+                                                      presimpStr);
+    vtkm::worklet::contourtree_augmented::PrintIndices("Volumes: ", *volumeArray, -1, presimpStr);
+    vtkm::worklet::contourtree_augmented::PrintHeader(baseTree->Superparents.GetNumberOfValues(),
+                                                      presimpStr);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "Global Regular", baseTree->RegularNodeGlobalIds, -1, presimpStr);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "Superparents", baseTree->Superparents, -1, presimpStr);
+  } // presimplifying
+  VTKM_LOG_S(vtkm::cont::LogLevel::Info, presimpStr.str());
+#endif
+
   // to do this, we construct an index array with all supernode ID's that satisfy:
   // 1. superparent == NO_SUCH_ELEMENT (i.e. root of interior tree)
   // 2. round < nRounds (except the top level, where 1. indicates the tree root)
@@ -294,6 +326,65 @@ void HierarchicalAugmenter<FieldType>::Initialize(
                                   presimplifyThreshold);
     auto tempSupernodeIndex =
       vtkm::cont::ArrayHandleIndex(this->BaseTree->Supernodes.GetNumberOfValues());
+
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+    vtkm::cont::Algorithm::Copy(tempSupernodeIndex, this->AttachmentIds);
+    std::cout << "Block: " << blockId << std::endl;
+    vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues(),
+                                                      std::cout);
+    vtkm::worklet::contourtree_augmented::PrintIndices("Attachment ID", this->AttachmentIds);
+
+    auto supernodes = this->BaseTree->Supernodes.ReadPortal();
+    auto superarcs = this->BaseTree->Superarcs.ReadPortal();
+    auto whichRound = this->BaseTree->WhichRound.ReadPortal();
+    auto volumeArr =
+      presimplify ? volumeArray->ReadPortal() : this->BaseTree->WhichRound.ReadPortal();
+    auto regNodeGlobalIds = this->BaseTree->RegularNodeGlobalIds.ReadPortal();
+    auto attachmentIds = this->AttachmentIds.WritePortal();
+    for (vtkm::Id supernode = 0; supernode < this->AttachmentIds.GetNumberOfValues(); supernode++)
+    { // per supernode
+      std::cout << "Processing supernode " << supernode << std::endl;
+      std::cout << "Regular ID           " << supernodes.Get(supernode) << std::endl;
+      std::cout << "Global Regular ID    " << regNodeGlobalIds.Get(supernodes.Get(supernode))
+                << std::endl;
+
+      // an attachment point is defined by having no superarc (NO_SUCH_ELEMENT) and not being in the final round (where this indicates the global root)
+      if (vtkm::worklet::contourtree_augmented::NoSuchElement(superarcs.Get(supernode)) &&
+          (whichRound.Get(supernode) < baseTree->NumRounds))
+      // passes the predicate - nothing further needed
+      { // passes the predicate
+        std::cout << "Attachment Point: it passed the first test" << std::endl;
+        if (presimplify)
+        {
+          std::cout << "Volume:      " << volumeArr.Get(supernode) << std::endl;
+          std::cout << "Threshold:   " << presimplifyThreshold << std::endl;
+        }
+
+        // suppress if it's volume is at or below the threshold
+        if (presimplify && volumeArr.Get(supernode) <= presimplifyThreshold)
+        { // below threshold
+          attachmentIds.Set(supernode, vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT);
+          std::cout << "Failed Second Test" << std::endl;
+        } // below threshold
+        else
+          std::cout << "Volume Greater than Threshold: it passed the second test" << std::endl;
+        std::cout << "Block: " << blockId << std::endl;
+        vtkm::worklet::contourtree_augmented::PrintHeader(attachmentIds.GetNumberOfValues());
+        vtkm::worklet::contourtree_augmented::PrintIndices("Attachment ID", this->AttachmentIds);
+      } // passes the predicate
+      else
+      { // fails
+        // reset the value
+        attachmentIds.Set(supernode, vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT);
+      } // fails
+    }   // per supernode
+
+    std::cout << "Block: " << blockId << std::endl;
+    vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues(),
+                                                      std::cout);
+    vtkm::worklet::contourtree_augmented::PrintIndices("Attachment ID", this->AttachmentIds);
+#endif
+
     vtkm::cont::Algorithm::CopyIf(
       // first a list of all of the supernodes
       tempSupernodeIndex,
@@ -307,6 +398,13 @@ void HierarchicalAugmenter<FieldType>::Initialize(
       // being in the final round (where this indicates the global root) defined by the condition
       // if (noSuchElement(baseTree->superarcs[supernode]) && (baseTree->whichRound[supernode] < baseTree->nRounds))
       isAttachementPointPredicate);
+
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+    std::cout << "Block: " << blockId << std::endl;
+    vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues(),
+                                                      std::cout);
+    vtkm::worklet::contourtree_augmented::PrintIndices("Attachment ID", this->AttachmentIds);
+#endif
   }
 
   // we now resize the working arrays
@@ -360,6 +458,19 @@ void HierarchicalAugmenter<FieldType>::Initialize(
     vtkm::cont::Algorithm::Copy(isAscendingSuperparentArr, this->Superparents);
   }
 
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << "Block: " << blockId << std::endl;
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintIndices("Attachment ID", this->AttachmentIds);
+  vtkm::worklet::contourtree_augmented::PrintIndices("Global Regular ID", this->GlobalRegularIds);
+  vtkm::worklet::contourtree_augmented::PrintValues<FieldType>("Data Value", this->DataValues);
+  vtkm::worklet::contourtree_augmented::PrintIndices("Supernode ID", this->SupernodeIds);
+  vtkm::worklet::contourtree_augmented::PrintIndices("Superparent ID ", this->Superparents);
+  vtkm::worklet::contourtree_augmented::PrintIndices("Superparent Round", this->SuperparentRounds);
+  vtkm::worklet::contourtree_augmented::PrintIndices("Which Round", this->WhichRounds);
+  std::cout << std::endl;
+#endif
+
   // clean up memory
   this->AttachmentIds.ReleaseResources();
 } // Initialize()
@@ -435,6 +546,12 @@ void HierarchicalAugmenter<FieldType>::RetrieveInAttachmentPoints()
   vtkm::Id numIncomingAttachments = InData.GlobalRegularIds.GetNumberOfValues();
   vtkm::Id numTotalAttachments = numAttachmentsCurrently + numIncomingAttachments;
 
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << "nAttachmentsCurrently: " << numAttachmentsCurrently << std::endl;
+  std::cout << "nIncomingAttachments:  " << numIncomingAttachments << std::endl;
+  std::cout << "nTotalAttachments:     " << numTotalAttachments << std::endl;
+#endif
+
   // I.  resize the existing arrays
   vtkm::worklet::contourtree_augmented::ResizeVector<vtkm::Id>(
     this->GlobalRegularIds, numTotalAttachments, static_cast<vtkm::Id>(0));
@@ -513,8 +630,22 @@ void HierarchicalAugmenter<FieldType>::BuildAugmentedTree()
 { // BuildAugmentedTree()
   // 1.  Prepare the data structures for filling in, copying in basic information & organising the attachment points
   this->PrepareAugmentedTree();
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Block ") + std::to_string(this->BlockId) +
+                            std::string(" Step 1: Augmented Tree Prepared"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
+#endif
   // 2.  Copy the hyperstructure, using the old super IDs for now
   this->CopyHyperstructure();
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Block ") + std::to_string(this->BlockId) +
+                            std::string(" Step 2: Hyperstructure Copied"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
+#endif
   // 3.	Copy superstructure one round at a time, updating the hyperstructure as well
   // (needed to permit search for superarcs)
   //	Loop from the top down:
@@ -546,6 +677,10 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
   //    segments with identical superparent round, which is all we need for now
   vtkm::cont::Algorithm::Copy(
     vtkm::cont::ArrayHandleIndex(this->GlobalRegularIds.GetNumberOfValues()), this->AttachmentIds);
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << this->DebugPrint("Attachment Points List Constructed", __FILE__, __LINE__)
+            << std::endl;
+#endif
   // 1a.  We now need to suppress duplicates,
   {
     // Sort the attachement Ids
@@ -553,10 +688,21 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
       AttachmentSuperparentAndIndexComparator attachmentSuperparentAndIndexComparator(
         this->SuperparentRounds, this->GlobalRegularIds, this->SupernodeIds);
     vtkm::cont::Algorithm::Sort(this->AttachmentIds, attachmentSuperparentAndIndexComparator);
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+    std::cout << this->DebugPrint(
+                   "Attachment Points Sorted on Superparent Round", __FILE__, __LINE__)
+              << std::endl;
+    const vtkm::Id nAttachmentDupIds = this->AttachmentIds.GetNumberOfValues();
+#endif
     // Remove the duplicate values using GlobalRegularIds[AttachmentIds] for checking for equality
     vtkm::worklet::contourtree_distributed::hierarchical_augmenter::AttachmentIdsEqualComparator
       attachmentIdsEqualComparator(this->GlobalRegularIds);
     vtkm::cont::Algorithm::Unique(this->AttachmentIds, attachmentIdsEqualComparator);
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+    std::cout << "Block " << this->BlockId << ": reducing attachment point list from size "
+              << nAttachmentDupIds << " to size " << this->AttachmentIds.GetNumberOfValues()
+              << std::endl;
+#endif
   }
 
   //  2.  Set up array with bounds for subsegments
@@ -567,6 +713,14 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
                                               this->BaseTree->NumRounds + 2),
     this->FirstAttachmentPointInRound);
 
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("FirstAttachment Array resized to ") +
+                            std::to_string(this->BaseTree->NumRounds + 2),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
+#endif
+
   // Now do a parallel set operation
   {
     vtkm::worklet::contourtree_distributed::hierarchical_augmenter::
@@ -584,9 +738,9 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
   firstAttachmentPointInRoundPortal.Set(this->BaseTree->NumRounds + 1,
                                         this->AttachmentIds.GetNumberOfValues());
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             this->DebugPrint("First Attachment Point Set Where Possible", __FILE__, __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << this->DebugPrint("First Attachment Point Set Where Possible", __FILE__, __LINE__)
+            << std::endl;
 #endif
   //     Now clean up by looping through the rounds (serially - this is logarithmic at worst)
   //     We loop backwards so that the next up propagates downwards
@@ -602,8 +756,8 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
     }
   } // per round
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info, DebugPrint("Subsegments Identified", __FILE__, __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint("Subsegments Identified", __FILE__, __LINE__) << std::endl;
 #endif
   //  3.  Initialise an array to keep track of the mapping from old supernode ID to new supernode ID
   vtkm::cont::Algorithm::Copy(
@@ -611,10 +765,8 @@ void HierarchicalAugmenter<FieldType>::PrepareAugmentedTree()
                                               this->BaseTree->Supernodes.GetNumberOfValues()),
     this->NewSupernodeIds);
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             this->DebugPrint("Augmented Tree Prepared", __FILE__, __LINE__));
-
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << this->DebugPrint("Augmented Tree Prepared", __FILE__, __LINE__) << std::endl;
 #endif
 } // PrepareAugmentedTree()
 
@@ -687,8 +839,8 @@ void HierarchicalAugmenter<FieldType>::CopyHyperstructure()
     0                                                  // set all elements to this value
   );
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info, DebugPrint("Hyperstructure Copied", __FILE__, __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint("Hyperstructure Copied", __FILE__, __LINE__) << std::endl;
 #endif
 } // CopyHyperstructure()
 
@@ -755,19 +907,35 @@ void HierarchicalAugmenter<FieldType>::UpdateHyperstructure(vtkm::Id roundNumber
 
   // finally, find the number of superchildren as the delta between the
   // super ID and the next hypernode's super ID
+  // This is slightly tricky. Multiple supernodes may share the same hyperparent.
   {
+    /* ----------- MINGZHE: Bug fix here ---------  */
+    // We cannot use FirstHypernodePerIteration[roundNumber].
+    vtkm::Id superchildrenStartIndex =
+      vtkm::cont::ArrayGetValue(0, this->AugmentedTree->FirstSupernodePerIteration[roundNumber]);
+    vtkm::Id superchildrenStopIndex = vtkm::cont::ArrayGetValue(
+      vtkm::cont::ArrayGetValue(roundNumber, this->AugmentedTree->NumIterations),
+      this->AugmentedTree->FirstSupernodePerIteration[roundNumber]);
+    vtkm::Id superchildrenSelectSize = superchildrenStopIndex - superchildrenStartIndex;
+    vtkm::Id extraSelectSize = ((superchildrenStartIndex + superchildrenSelectSize) <
+                                this->AugmentedTree->Hyperparents.GetNumberOfValues())
+      ? superchildrenSelectSize + 1
+      : superchildrenSelectSize;
+
+    // WARNING! WARNING! WARNING!
+    /* ----------- MINGZHE: Bug fix here ---------  */
+    // Because we use ArrayHandleView to select the size of array for the array,
+    // the index of the entry in the worklet is NOT the actual array index.
+    // We need to send the starting index of supernodes into the worklet.
     vtkm::worklet::contourtree_distributed::hierarchical_augmenter::
       UpdateHyperstructureSetSuperchildrenWorklet updateHyperstructureSetSuperchildrenWorklet(
-        this->AugmentedTree->Supernodes.GetNumberOfValues());
+        this->AugmentedTree->Supernodes.GetNumberOfValues(), superchildrenStartIndex);
     // As above, we need to create views of the relevant subranges of our arrays
-    auto augmentedTreeSuperarcsView =
-      vtkm::cont::make_ArrayHandleView(this->AugmentedTree->Superarcs, startIndex, selectSize);
-    vtkm::Id extraSelectSize =
-      ((startIndex + selectSize) < this->AugmentedTree->Hyperparents.GetNumberOfValues())
-      ? selectSize + 1
-      : selectSize;
+    auto augmentedTreeSuperarcsView = vtkm::cont::make_ArrayHandleView(
+      this->AugmentedTree->Superarcs, superchildrenStartIndex, superchildrenSelectSize);
     auto augmentedTreeHyperparentsView = vtkm::cont::make_ArrayHandleView(
-      this->AugmentedTree->Hyperparents, startIndex, extraSelectSize);
+      this->AugmentedTree->Hyperparents, superchildrenStartIndex, extraSelectSize);
+
     this->Invoke(updateHyperstructureSetSuperchildrenWorklet,
                  this->AugmentedTree->Hypernodes,   // input
                  augmentedTreeSuperarcsView,        // input
@@ -775,6 +943,9 @@ void HierarchicalAugmenter<FieldType>::UpdateHyperstructure(vtkm::Id roundNumber
                  this->AugmentedTree->Superchildren // output
     );
   }
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << this->DebugPrint("Hyperstructure Updated", __FILE__, __LINE__) << std::endl;
+#endif
 } // UpdateHyperstructure()
 
 
@@ -961,6 +1132,7 @@ void HierarchicalAugmenter<FieldType>::RetrieveOldSupernodes(vtkm::Id roundNumbe
     supernodeIndexBase, // start
     1,                  // step
     this->BaseTree->NumSupernodesInRound.ReadPortal().Get(roundNumber));
+
   {
     // Reset this-KeptSupernodes to the right size and initalize with NO_SUCH_ELEMENT.
     vtkm::cont::Algorithm::Copy(
@@ -970,8 +1142,9 @@ void HierarchicalAugmenter<FieldType>::RetrieveOldSupernodes(vtkm::Id roundNumbe
         this->BaseTree->NumSupernodesInRound.ReadPortal().Get(roundNumber)),
       // target array
       this->KeptSupernodes);
+
     // Create the predicate for the CopyIf
-    vtkm::worklet::contourtree_augmented::NotNoSuchElementPredicate notNoSuchElementPredicate;
+    vtkm::worklet::contourtree_augmented::NoSuchElementPredicate NoSuchElementPredicate;
     // Copy supernodeId to this->KeptSupernodes
     vtkm::cont::Algorithm::CopyIf(
       // first we generate a list of supernodeIds
@@ -986,15 +1159,15 @@ void HierarchicalAugmenter<FieldType>::RetrieveOldSupernodes(vtkm::Id roundNumbe
       // are all points that suffice the condition
       // vtkm::Id supernodeID = keptSupernode + supernodeIndexBase;
       // !noSuchElement(baseTree->superarcs[supernodeID]);
-      notNoSuchElementPredicate);
+      NoSuchElementPredicate);
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Old Supernodes Retrieved"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Old Supernodes Retrieved"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 } // RetrieveOldSupernodes()
 
@@ -1080,12 +1253,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
       vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT);
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(
-    vtkm::cont::LogLevel::Info,
-    DebugPrint(std::string("Round ") + std::to_string(roundNumber) + std::string(" Arrays Resized"),
-               __FILE__,
-               __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Arrays Resized"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   // The next task is to assemble a sorting array which we will use to construct the new superarcs, containing both the
@@ -1104,12 +1277,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
     vtkm::worklet::contourtree_augmented::ResizeVector<vtkm::Id>(
       this->SupernodeIdSet, numSupernodesThisLevel, static_cast<vtkm::Id>(0));
   }
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Sorter Set Resized"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Sorter Set Resized"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   // b. Transfer attachment points for level into new supernode array
@@ -1156,12 +1329,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
       supernodeIdsPermuted, 0, supernodeIdsPermuted.GetNumberOfValues(), this->SupernodeIdSet, 0);
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Attachment Points Transferred"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Attachment Points Transferred"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   // Now we copy in the kept supernodes: this used to mean only the non-attachment points
@@ -1210,12 +1383,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
                                         numInsertedSupernodes);
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Kept Supernodes Transferred"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Kept Supernodes Transferred"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   //  c.  Create a permutation array and sort supernode segment by a. superparent, b. value, d. global index to establish segments (reversing as needed)
@@ -1227,12 +1400,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
     vtkm::cont::Algorithm::Sort(this->SupernodeSorter, attachmentAndSupernodeComparator);
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Sorter Set Sorted"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Sorter Set Sorted"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   //  d.  Build the inverse permutation array for lookup purposes:
@@ -1261,12 +1434,12 @@ void HierarchicalAugmenter<FieldType>::ResizeArrays(vtkm::Id roundNumber)
     // Add baseTree->regular2supernode
   }
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Sorting Arrays Built"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Sorting Arrays Built"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 } // ResizeArrays()
 
@@ -1276,12 +1449,12 @@ template <typename FieldType>
 void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
 { // CreateSuperarcs()
   // retrieve the ID number of the first supernode at this level
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Starting CreateSuperarcs()"),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Starting CreateSuperarcs()"),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   vtkm::Id currNumIterations =
@@ -1321,20 +1494,19 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
         numSupernodesAlready, this->BaseTree->NumRounds, numInsertedSupernodes, roundNumber);
 
     // create fancy arrays needed to allow use of FieldIn for worklet parameters
+
+    // WARNING! WARNING! WARNING!
+    /* ----------- MINGZHE: Bug fix here ---------  */
+    // permutedSupernodeIdSet may be NO_SUCH_ELEMENT if not already in the base tree
+    // We cannot use it for permutation index
+    // For any array using permutedSupernodeIdSet as indices, we put them into the data exec object.
     auto permutedSupernodeIdSet =
       vtkm::cont::make_ArrayHandlePermutation(this->SupernodeSorter, this->SupernodeIdSet);
-    auto permutedBaseTreeSuper2Hypernode = vtkm::cont::make_ArrayHandlePermutation(
-      permutedSupernodeIdSet,
-      this->BaseTree
-        ->Super2Hypernode); // Get this->BaseTree->Super2hypernode[oldSupernodeId]; as FieldIn for the worklet
+
     auto permutedGlobalRegularIdSet =
       vtkm::cont::make_ArrayHandlePermutation(this->SupernodeSorter, this->GlobalRegularIdSet);
     auto permutedDataValueSet =
       vtkm::cont::make_ArrayHandlePermutation(this->SupernodeSorter, this->DataValueSet);
-    auto oldRegularId =
-      vtkm::cont::make_ArrayHandlePermutation(permutedSupernodeIdSet, this->BaseTree->Supernodes);
-    auto oldSuperFrom =
-      vtkm::cont::make_ArrayHandlePermutation(oldRegularId, this->BaseTree->Superparents);
 
     // Create a view of the range of this->AugmentedTree->Superarcs that will be updated by the worklet
     // so that we can use FieldOut as the type in the worklet
@@ -1343,6 +1515,7 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
       numSupernodesAlready,                     // start view here
       this->SupernodeSorter.GetNumberOfValues() // select this many values
     );
+
     auto augmentedTreeHyperparentsView = vtkm::cont::make_ArrayHandleView(
       this->AugmentedTree->Hyperparents,
       numSupernodesAlready,                     // start view here
@@ -1393,7 +1566,10 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
         this->BaseTree->Hyperparents,
         this->BaseTree->WhichRound,
         this->BaseTree->WhichIteration,
+        this->BaseTree->Supernodes,
         this->BaseTree->Superarcs,
+        this->BaseTree->Superparents,
+        this->BaseTree->Super2Hypernode,
         this->BaseTree->Hypernodes,
         this->SuperparentSet,
         this->NewSupernodeIds);
@@ -1401,14 +1577,12 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
     // invoke the worklet
     this->Invoke(createSuperarcsWorklet, // the worklet
                  // inputs
-                 this->SupernodeSorter,           // input domain (WholeArrayIn)
-                 permutedSupernodeIdSet,          // input (FieldIn)
-                 permutedBaseTreeSuper2Hypernode, // input (FieldIn)
-                 permutedGlobalRegularIdSet,      // input (FieldIn)
-                 permutedDataValueSet,            // input (FieldIn)
-                 oldSuperFrom,                    // input (FieldIn)
-                 findSuperArcForUnknownNode,      // input (Execution object)
-                 createSuperarcsDataExecObj,      // input (Execution object with BaseTreeData
+                 this->SupernodeSorter,      // input domain (WholeArrayIn)
+                 permutedSupernodeIdSet,     // input (FieldIn)
+                 permutedGlobalRegularIdSet, // input (FieldIn)
+                 permutedDataValueSet,       // input (FieldIn)
+                 findSuperArcForUnknownNode, // input (Execution object)
+                 createSuperarcsDataExecObj, // input (Execution object with BaseTreeData
                  // Outputs
                  this->AugmentedTree->Supernodes,       // input/output
                  augmentedTreeSuperarcsView,            // output
@@ -1421,15 +1595,14 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
                  augmentedTreeRegular2SupernodeView,    // output
                  augmentedTreeSuperparentsView          // output
     );
-
   } // END Call CreateSuperarcsWorklet
 
-#ifdef DEBUG_PRINT
-  VTKM_LOG_S(vtkm::cont::LogLevel::Info,
-             DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
-                          std::string(" Details Filled in For Supernodes "),
-                        __FILE__,
-                        __LINE__));
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+  std::cout << DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
+                            std::string(" Details Filled in For Supernodes "),
+                          __FILE__,
+                          __LINE__)
+            << std::endl;
 #endif
 
   // Now, in order to set the first supernode per iteration, we do an additional loop
@@ -1454,7 +1627,6 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
     this->AugmentedTree->Supernodes.GetNumberOfValues(),         // new value
     this->AugmentedTree->FirstSupernodePerIteration[roundNumber] // array
   );
-
   // The following loop should be safe in parallel since there should never be two zeros in sequence, i.e., the next
   // entry after a zero will always be valid, regardless of execution order
   // This was added because in rare cases there are no supernodes transferred in an iteration, for example because there
@@ -1504,15 +1676,18 @@ void HierarchicalAugmenter<FieldType>::CreateSuperarcs(vtkm::Id roundNumber)
         this->AugmentedTree->Supernodes.GetNumberOfValues(),         // new value
         this->AugmentedTree->FirstSupernodePerIteration[roundNumber] // array
       );
+
       // for the hypernode array, the last iteration is guaranteed not to have hyperarcs by construction
       // so the last iteration will already have the correct sentinel value, and we just need to shrink the array
-      this->AugmentedTree->FirstSupernodePerIteration[roundNumber].Allocate(
+      /* ----------- MINGZHE: Bug fix here ---------  */
+      // Should be FirstHypernodePerIteration instead of FirstSupernodePerIteration
+      this->AugmentedTree->FirstHypernodePerIteration[roundNumber].Allocate(
         iterationArraySize, vtkm::CopyFlag::On); // shrink array but keep values
     }                                            // attachment point round was removed
   }                                              // at least one iteration
 
 
-#ifdef DEBUG_PRINT
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
   VTKM_LOG_S(vtkm::cont::LogLevel::Info,
              DebugPrint(std::string("Round ") + std::to_string(roundNumber) +
                           std::string(" Superarcs Created "),
@@ -1537,7 +1712,7 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
                                                          long lineNum)
 { // DebugPrint()
   std::stringstream resultStream;
-  resultStream << std::endl;
+  resultStream << "%" << std::endl;
   resultStream << "----------------------------------------" << std::endl;
   resultStream << std::setw(30) << std::left << fileName << ":" << std::right << std::setw(4)
                << lineNum << std::endl;
@@ -1545,13 +1720,16 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
                << std::endl;
   resultStream << "----------------------------------------" << std::endl;
 
+#ifdef DEBUG_PRINT_HIERARCHICAL_CONTOUR_TREE
   resultStream << this->BaseTree->DebugPrint(
     (message + std::string(" Base Tree")).c_str(), fileName, lineNum);
   resultStream << this->AugmentedTree->DebugPrint(
     (message + std::string(" Augmented Tree")).c_str(), fileName, lineNum);
+#endif
   resultStream << "========================================" << std::endl;
   resultStream << "Local List of Attachment Points" << std::endl;
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->GlobalRegularIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->GlobalRegularIds.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Global Regular Ids", this->GlobalRegularIds, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintValues(
@@ -1567,7 +1745,7 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
   resultStream << std::endl;
   resultStream << "Outgoing Attachment Points" << std::endl;
   vtkm::worklet::contourtree_augmented::PrintHeader(
-    this->OutData.GlobalRegularIds.GetNumberOfValues());
+    this->OutData.GlobalRegularIds.GetNumberOfValues(), resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Out Global Regular Ids", this->OutData.GlobalRegularIds, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintValues(
@@ -1581,37 +1759,45 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Out WhichRounds", this->OutData.WhichRounds, -1, resultStream);
   resultStream << std::endl;
-  resultStream << "Incoming Attachment Points" << std::endl;
-  vtkm::worklet::contourtree_augmented::PrintHeader(
-    this->InData.GlobalRegularIds.GetNumberOfValues());
-  vtkm::worklet::contourtree_augmented::PrintIndices(
-    "In Global Regular Ids", this->InData.GlobalRegularIds, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintValues(
-    "In Data Values", this->InData.DataValues, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintIndices(
-    "In Supernode Ids", this->InData.SupernodeIds, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintIndices(
-    "In Superparents", this->InData.Superparents, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintIndices(
-    "In Superparent Rounds", this->InData.SuperparentRounds, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintIndices(
-    "In WhichRounds", this->InData.WhichRounds, -1, resultStream);
-  resultStream << std::endl;
+  // we only output the incoming attachment points in the data exchange debug print
+  if (message.find(std::string("In Attachment Points Received")) != std::string::npos)
+  {
+    resultStream << "Incoming Attachment Points" << std::endl;
+    vtkm::worklet::contourtree_augmented::PrintHeader(
+      this->InData.GlobalRegularIds.GetNumberOfValues(), resultStream);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "In Global Regular Ids", this->InData.GlobalRegularIds, -1, resultStream);
+    vtkm::worklet::contourtree_augmented::PrintValues(
+      "In Data Values", this->InData.DataValues, -1, resultStream);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "In Supernode Ids", this->InData.SupernodeIds, -1, resultStream);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "In Superparents", this->InData.Superparents, -1, resultStream);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "In Superparent Rounds", this->InData.SuperparentRounds, -1, resultStream);
+    vtkm::worklet::contourtree_augmented::PrintIndices(
+      "In WhichRounds", this->InData.WhichRounds, -1, resultStream);
+    resultStream << std::endl;
+  }
   resultStream << "Holding Arrays" << std::endl;
   vtkm::worklet::contourtree_augmented::PrintHeader(
-    this->FirstAttachmentPointInRound.GetNumberOfValues());
+    this->FirstAttachmentPointInRound.GetNumberOfValues(), resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "First Attach / Rd", this->FirstAttachmentPointInRound, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->AttachmentIds.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "AttachmentIds", this->AttachmentIds, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->NewSupernodeIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->NewSupernodeIds.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "New Supernode Ids", this->NewSupernodeIds, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->KeptSupernodes.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->KeptSupernodes.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Kept Supernodes", this->KeptSupernodes, -1, resultStream);
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->SupernodeSorter.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->SupernodeSorter.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Supernode Sorter", this->SupernodeSorter, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
@@ -1625,7 +1811,8 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
   resultStream << std::endl;
   resultStream << std::endl;
 
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->SupernodeSorter.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->SupernodeSorter.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "Supernode Id", this->SupernodeSorter, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintArrayHandle(
@@ -1651,7 +1838,8 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
   resultStream << std::endl;
   resultStream << std::endl;
 
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->RegularSuperparents.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(this->RegularSuperparents.GetNumberOfValues(),
+                                                    resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
     "RegularNodesNeeded", this->RegularNodesNeeded, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(

vtkm/filter/scalar_topology/worklet/contourtree_distributed/HierarchicalAugmenterFunctor.h

@@ -64,7 +64,6 @@ VTKM_THIRDPARTY_PRE_INCLUDE
 VTKM_THIRDPARTY_POST_INCLUDE
 // clang-format on
 
-
 namespace vtkm
 {
 namespace worklet
@@ -109,8 +108,20 @@ public:
       int ingid = rp.in_link().target(i).gid;
       if (ingid != selfid)
       { // Receive and augment
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+        std::cout << "Block   " << selfid << std::endl;
+        std::cout << "Partner " << ingid << std::endl;
+#endif
         rp.dequeue(ingid, blockData->HierarchicalAugmenter.InData);
         blockData->HierarchicalAugmenter.RetrieveInAttachmentPoints();
+#ifdef DEBUG_HIERARCHICAL_AUGMENTER
+        std::cout << blockData->HierarchicalAugmenter.DebugPrint(
+                       std::string("Round ") + std::to_string(round - 1) +
+                         std::string(" In Attachment Points Received"),
+                       __FILE__,
+                       __LINE__)
+                  << std::endl;
+#endif
       }
     }
 

vtkm/filter/scalar_topology/worklet/contourtree_distributed/hierarchical_augmenter/CreateSuperarcsData.h

@@ -84,7 +84,10 @@ public:
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeHyperparents,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeWhichRound,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeWhichIteration,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSupernodes,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuperarcs,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuperparents,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuper2Hypernode,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeHypernodes,
     const vtkm::worklet::contourtree_augmented::IdArrayType& superparentSet,
     const vtkm::worklet::contourtree_augmented::IdArrayType& newSupernodeIds,
@@ -94,7 +97,10 @@ public:
     this->BaseTreeHyperparents = baseTreeHyperparents.PrepareForInput(device, token);
     this->BaseTreeWhichRound = baseTreeWhichRound.PrepareForInput(device, token);
     this->BaseTreeWhichIteration = baseTreeWhichIteration.PrepareForInput(device, token);
+    this->BaseTreeSupernodes = baseTreeSupernodes.PrepareForInput(device, token);
     this->BaseTreeSuperarcs = baseTreeSuperarcs.PrepareForInput(device, token);
+    this->BaseTreeSuperparents = baseTreeSuperparents.PrepareForInput(device, token);
+    this->BaseTreeSuper2Hypernode = baseTreeSuper2Hypernode.PrepareForInput(device, token);
     this->BaseTreeHypernodes = baseTreeHypernodes.PrepareForInput(device, token);
     this->SuperparentSet = superparentSet.PrepareForInput(device, token);
     this->NewSupernodeIds = newSupernodeIds.PrepareForInput(device, token);
@@ -104,7 +110,10 @@ public:
   IndicesPortalType BaseTreeHyperparents;
   IndicesPortalType BaseTreeWhichRound;
   IndicesPortalType BaseTreeWhichIteration;
+  IndicesPortalType BaseTreeSupernodes;
   IndicesPortalType BaseTreeSuperarcs;
+  IndicesPortalType BaseTreeSuperparents;
+  IndicesPortalType BaseTreeSuper2Hypernode;
   IndicesPortalType BaseTreeHypernodes;
   IndicesPortalType SuperparentSet;
   IndicesPortalType NewSupernodeIds;
@@ -119,14 +128,20 @@ public:
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeHyperparents,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeWhichRound,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeWhichIteration,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSupernodes,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuperarcs,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuperparents,
+    const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeSuper2Hypernode,
     const vtkm::worklet::contourtree_augmented::IdArrayType& baseTreeHypernodes,
     const vtkm::worklet::contourtree_augmented::IdArrayType& superparentSet,
     const vtkm::worklet::contourtree_augmented::IdArrayType& newSupernodeIds)
     : BaseTreeHyperparents(baseTreeHyperparents)
     , BaseTreeWhichRound(baseTreeWhichRound)
     , BaseTreeWhichIteration(baseTreeWhichIteration)
+    , BaseTreeSupernodes(baseTreeSupernodes)
     , BaseTreeSuperarcs(baseTreeSuperarcs)
+    , BaseTreeSuperparents(baseTreeSuperparents)
+    , BaseTreeSuper2Hypernode(baseTreeSuper2Hypernode)
     , BaseTreeHypernodes(baseTreeHypernodes)
     , SuperparentSet(superparentSet)
     , NewSupernodeIds(newSupernodeIds)
@@ -140,7 +155,10 @@ public:
     return CreateSuperarcsData(BaseTreeHyperparents,
                                BaseTreeWhichRound,
                                BaseTreeWhichIteration,
+                               BaseTreeSupernodes,
                                BaseTreeSuperarcs,
+                               BaseTreeSuperparents,
+                               BaseTreeSuper2Hypernode,
                                BaseTreeHypernodes,
                                SuperparentSet,
                                NewSupernodeIds,
@@ -153,7 +171,10 @@ private:
   const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeHyperparents;
   const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeWhichRound;
   const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeWhichIteration;
+  const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeSupernodes;
   const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeSuperarcs;
+  const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeSuperparents;
+  const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeSuper2Hypernode;
   const vtkm::worklet::contourtree_augmented::IdArrayType& BaseTreeHypernodes;
   const vtkm::worklet::contourtree_augmented::IdArrayType& SuperparentSet;
   const vtkm::worklet::contourtree_augmented::IdArrayType& NewSupernodeIds;

vtkm/filter/scalar_topology/worklet/contourtree_distributed/hierarchical_augmenter/CreateSuperarcsWorklet.h

@@ -72,18 +72,12 @@ public:
   ///                           therefore this is a WholeArrayIn transfer.
   /// @param[in] supernodeIdSetPermuted Field in of supernodeIdSet permuted by the supernodeSorter array to
   ///                                 allow us to use FieldIn
-  /// @param[in] baseTreeHyperparents whole array input of the BaseTree->Hyperparents
-  /// @param[in] baseTreeSuper2HypernodePermuted  baseTreeSuper2Hypernode permuted by supernodeIdSetPermuted in order
-  ///                          to extract baseTree->super2hypernode[oldSupernodeID];
-  /// @param[in] baseTreeWhichRound
-  /// @param[in] baseTreeWhichIteration
   /// @param[in] globalRegularIdSetPermuted Field in of globalRegularIdSet permuted by supernodeSorter array to
   ///                           allow use of FieldIn
   /// @param[in] dataValueSetPermuted Field in of dataValyeSet permuted by supernodeSorter array to
   ///                           allow use of FieldIn
-  /// @param[in] baseTreeSuperarcs  BaseTree->Superarcs
-  /// @param[in] oldSuperFrom Permuted baseTree->SuperParents[  baseTree->Supernodes[ supernodeIdSetPermuted  ]  ]
-  /// @param[in] baseTreeHypernodes
+  /// @param[in] ExecObject findSuperArcForUnknownNode Execute object in to find the superarc of arbitrary node
+  /// @param[in] ExecObject createSuperarcsData Data object in, storing many BaseTree arrays
   /// @param[out, in] augmentedTreeSupernodes  this->AugmentedTree->Supernodes array
   /// @param[out] augmentedTreeSuperarcsView  output view of  this->AugmentedTree->Superarcs with
   ///                           vtkm::cont::make_ArrayHandleView(this->AugmentedTree->Superarcs,
@@ -135,11 +129,8 @@ public:
     // Inputs
     WholeArrayIn supernodeSorter,
     FieldIn supernodeIdSetPermuted,
-    FieldIn baseTreeSuper2HypernodePermuted,
     FieldIn globalRegularIdSetPermuted,
     FieldIn dataValueSetPermuted,
-    FieldIn
-      oldSuperFrom, // baseTree->SuperParents[  baseTree->Supernodes[ supernodeIdSetPermuted  ]  ]
     ExecObject findSuperArcForUnknownNode,
     ExecObject createSuperarcsData,
     // Outputs
@@ -153,26 +144,9 @@ public:
     FieldOut augmentedTreeDataValuesView,
     FieldOut augmentedTreeRegular2SupernodeView,
     FieldOut augmentedTreeSuperparentsViews);
-  using ExecutionSignature = void(InputIndex,
-                                  _1,
-                                  _2,
-                                  _3,
-                                  _4,
-                                  _5,
-                                  _6,
-                                  _7,
-                                  _8,
-                                  _9,
-                                  _10,
-                                  _11,
-                                  _12,
-                                  _13,
-                                  _14,
-                                  _15,
-                                  _16,
-                                  _17,
-                                  _18);
-  // using ExecutionSignature = void(InputIndex, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20);
+  using ExecutionSignature =
+    void(InputIndex, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16);
+
   using InputDomain = _1;
 
   /// Default Constructor
@@ -205,13 +179,9 @@ public:
     const vtkm::Id& supernode, // InputIndex of supernodeSorter
     const InFieldPortalType& supernodeSorterPortal,
     const vtkm::Id& oldSupernodeId, //  supernodeIdSet[supernodeSorterPortal.Get(supernode)]
-    const vtkm::Id&
-      baseTreeSuper2HypernodeOldSupernodeIdValue, // baseTree->super2hypernode[oldSupernodeID];
     const vtkm::Id&
       globalRegularIdSetValue, //  globalRegularIdSet[supernodeSorterPortal.Get(supernode)]];
     const FieldType& dataValueSetValue, //  dataValueSet[supernodeSorterPortal.Get(supernode)]];
-    const vtkm::Id&
-      oldSuperFromValue, // baseTree->SuperParents[  baseTree->Supernodes[ supernodeIdSetPermuted  ]  ]
     const ExecObjType&
       findSuperArcForUnknownNode, // Execution object to call FindSuperArcForUnknownNode
     const ExecObjectTypeData&
@@ -273,7 +243,8 @@ public:
       // hyperstructure carries over, so we use the same hyperparent as before
       augmentedTreeHyperparentsValue = createSuperarcsData.BaseTreeHyperparents.Get(oldSupernodeId);
       // and set the hypernode ID
-      augmentedTreeSuper2HypernodeValue = baseTreeSuper2HypernodeOldSupernodeIdValue;
+      augmentedTreeSuper2HypernodeValue =
+        createSuperarcsData.BaseTreeSuper2Hypernode.Get(oldSupernodeId);
       // and the round and iteration
       augmentedTreeWhichRoundValue = createSuperarcsData.BaseTreeWhichRound.Get(oldSupernodeId);
       augmentedTreeWhichIterationValue =
@@ -296,6 +267,9 @@ public:
       // set the superarc to NO_SUCH_ELEMENT, as before
       augmentedTreeSuperarcsValue = vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT;
       // hyperstructure carries over, so we use the same hyperparent as before
+      // WARNING! WARNING! WARNING!
+      // the "if" clauses guarantee the oldSupernodeId not to be NO_SUCH_ELEMENT.
+      // We cannot use array permutation outside the worklet, or the guarantee does not hold!
       augmentedTreeHyperparentsValue = createSuperarcsData.BaseTreeHyperparents.Get(oldSupernodeId);
       // attachment points are never hypernodes anyway, so set it directly
       augmentedTreeSuper2HypernodeValue = vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT;
@@ -320,10 +294,15 @@ public:
       // we can guarantee that both ends of the parent are in the higher levels.  Which means we only need to work
       // out which end is higher.
 
-      // oldSuperFrom is provided as input and extracted as FieldIn on call. oldRegularId is not needed here
+      // WARNING! WARNING! WARNING!
+      /* ----------- MINGZHE: Bug fix here ---------  */
+      // the "if" clauses guarantee the oldSupernodeId not to be NO_SUCH_ELEMENT.
+      // We cannot use array permutation outside the worklet, or the guarantee does not hold!
       //      indexType oldRegularID = baseTree->supernodes[oldSupernodeID];
       //      indexType oldSuperFrom = baseTree->superparents[oldRegularID];
       //      indexType oldSuperTo = baseTree->superarcs[oldSuperFrom];
+      vtkm::Id oldRegularId = createSuperarcsData.BaseTreeSupernodes.Get(oldSupernodeId);
+      vtkm::Id oldSuperFromValue = createSuperarcsData.BaseTreeSuperparents.Get(oldRegularId);
       vtkm::Id oldSuperToValue = createSuperarcsData.BaseTreeSuperarcs.Get(oldSuperFromValue);
 
       // retrieve the ascending flag
@@ -368,7 +347,9 @@ public:
       // Since all of the superparents must be in the base tree, we can now retrieve the target
       vtkm::Id superparentOldSuperId = vtkm::worklet::contourtree_augmented::MaskedIndex(
         createSuperarcsData.SuperparentSet.Get(supernodeSetIndex));
+
       vtkm::Id oldTargetSuperId = createSuperarcsData.BaseTreeSuperarcs.Get(superparentOldSuperId);
+
       // and break it into a target and flags
       bool ascendingSuperarc = vtkm::worklet::contourtree_augmented::IsAscending(oldTargetSuperId);
       // NOTE: if the target was NO_SUCH_ELEMENT, this will hold 0

vtkm/filter/scalar_topology/worklet/contourtree_distributed/hierarchical_augmenter/UpdateHyperstructureSetSuperchildrenWorklet.h

@@ -72,12 +72,14 @@ public:
     WholeArrayInOut augmentedTreeSuperchildren // output
   );
   using ExecutionSignature = void(InputIndex, _1, _2, _3, _4);
-  using InputDomain = _1;
+  using InputDomain = _2;
 
   // Default Constructor
   VTKM_EXEC_CONT
-  UpdateHyperstructureSetSuperchildrenWorklet(const vtkm::Id& augmentedTreeNumSupernodes)
+  UpdateHyperstructureSetSuperchildrenWorklet(const vtkm::Id& augmentedTreeNumSupernodes,
+                                              const vtkm::Id& supernodeStartIndex)
     : AugmentedTreeNumSupernodes(augmentedTreeNumSupernodes)
+    , AugmentedTreeSupernodeStartIndex(supernodeStartIndex)
   {
   }
 
@@ -100,7 +102,7 @@ public:
     vtkm::Id hyperparentSuperId = augmentedTreeHypernodesPortal.Get(hyperparent);
 
     // we could be at the end of the array, so test explicitly
-    if (supernode == this->AugmentedTreeNumSupernodes - 1)
+    if (this->AugmentedTreeSupernodeStartIndex + supernode == this->AugmentedTreeNumSupernodes - 1)
     {
       // this means that we are the end of the segment and can subtract the hyperparent's super ID to get the number of superchildren
       augmentedTreeSuperchildrenPortal.Set(hyperparent,
@@ -111,7 +113,7 @@ public:
     {
       // again, subtract to get the number
       augmentedTreeSuperchildrenPortal.Set(
-        hyperparent, this->AugmentedTreeNumSupernodes + 1 - hyperparentSuperId);
+        hyperparent, supernode + this->AugmentedTreeSupernodeStartIndex + 1 - hyperparentSuperId);
     }
     // per supernode
     // In serial this worklet implements the following operation
@@ -139,6 +141,7 @@ public:
 
 private:
   const vtkm::Id AugmentedTreeNumSupernodes;
+  const vtkm::Id AugmentedTreeSupernodeStartIndex;
 }; // UpdateHyperstructureSetSuperchildrenWorklet
 
 } // namespace hierarchical_augmenter