Port changes to HierarchicalAugmenter to add In/Out arrays for MPI exchange

vtkm/filter/ContourTreeUniformDistributed.hxx

@@ -980,9 +980,8 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
           int ingid = rp.in_link().target(i).gid;
           if (ingid == selfid)
           { // Receive and augment
-            worklet::contourtree_distributed::HierarchicalAugmenter<FieldType> inAugmenter;
+            rp.dequeue(ingid, blockData->HierarchicalAugmenter.InData);
-            rp.dequeue(ingid, inAugmenter);
+            blockData->HierarchicalAugmenter.RetrieveInAttachmentPoints();
-            blockData->HierarchicalAugmenter.RetrieveInAttachmentPoints(inAugmenter);
           }
         }
 
@@ -993,8 +992,8 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
           { // Send to partner
             blockData->HierarchicalAugmenter.PrepareOutAttachmentPoints(round);
             // TODO/FIXME: Correct function? Correct round?
-            rp.enqueue(target, blockData->HierarchicalAugmenter);
+            rp.enqueue(target, blockData->HierarchicalAugmenter.OutData);
-            blockData->HierarchicalAugmenter.ReleaseOutArrays(); // TODO/FIXME: Correct function?
+            blockData->HierarchicalAugmenter.ReleaseSwapArrays();
           }
         }
       });

vtkm/worklet/contourtree_distributed/HierarchicalAugmenter.h

@@ -101,6 +101,7 @@
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/CopyBaseRegularStructureWorklet.h>
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/CreateSuperarcsWorklet.h>
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/FindSuperparentForNecessaryNodesWorklet.h>
+#include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/HierarchicalAugmenterInOutData.h>
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/IsAscendingDecorator.h>
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/IsAttachementPointNeededPredicate.h>
 #include <vtkm/worklet/contourtree_distributed/hierarchical_augmenter/IsAttachementPointPredicate.h>
@@ -160,12 +161,14 @@ public:
   /// if we're not careful, we'll have read-write conflicts when swapping with the partner
   /// there are other solutions, but the simpler solution is to have a transfer buffer for
   /// the set we want to send - which means another set of parallel arrays
-  vtkm::worklet::contourtree_augmented::IdArrayType OutGlobalRegularIds;
+  /// Output arrays used in DIY exchange
-  vtkm::cont::ArrayHandle<FieldType> OutDataValues;
+  vtkm::worklet::contourtree_distributed::hierarchical_augmenter::HierarchicalAugmenterInOutData<
-  vtkm::worklet::contourtree_augmented::IdArrayType OutSupernodeIds;
+    FieldType>
-  vtkm::worklet::contourtree_augmented::IdArrayType OutSuperparents;
+    OutData;
-  vtkm::worklet::contourtree_augmented::IdArrayType OutSuperparentRounds;
+  /// Output arrays used in DIY exchange
-  vtkm::worklet::contourtree_augmented::IdArrayType OutWhichRounds;
+  vtkm::worklet::contourtree_distributed::hierarchical_augmenter::HierarchicalAugmenterInOutData<
+    FieldType>
+    InData;
 
   /// these are essentially temporary local variables, but are placed here to make the DebugPrint()
   /// more comprehensive. They will be allocated where used
@@ -200,10 +203,10 @@ public:
   void PrepareOutAttachmentPoints(vtkm::Id round);
 
   /// routine to retrieve partner's current list of attachment points
-  void RetrieveInAttachmentPoints(HierarchicalAugmenter& partner);
+  void RetrieveInAttachmentPoints();
 
-  /// routine to release memory used for out arrays
+  /// routine to release memory used for swap arrays
-  void ReleaseOutArrays();
+  void ReleaseSwapArrays();
 
   /// routine to reconstruct a hierarchical tree using the augmenting supernodes
   void BuildAugmentedTree();
@@ -357,39 +360,39 @@ void HierarchicalAugmenter<FieldType>::PrepareOutAttachmentPoints(vtkm::Id round
   }
 
   //  4.  resize the out array
-  this->OutGlobalRegularIds.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.GlobalRegularIds.Allocate(this->AttachmentIds.GetNumberOfValues());
-  this->OutDataValues.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.DataValues.Allocate(this->AttachmentIds.GetNumberOfValues());
-  this->OutSupernodeIds.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.SupernodeIds.Allocate(this->AttachmentIds.GetNumberOfValues());
-  this->OutSuperparents.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.Superparents.Allocate(this->AttachmentIds.GetNumberOfValues());
-  this->OutSuperparentRounds.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.SuperparentRounds.Allocate(this->AttachmentIds.GetNumberOfValues());
-  this->OutWhichRounds.Allocate(this->AttachmentIds.GetNumberOfValues());
+  this->OutData.WhichRounds.Allocate(this->AttachmentIds.GetNumberOfValues());
 
   //  5.  copy the points we want
   {
     // outGlobalRegularIDs[outAttachmentPoint]  =  globalRegularIDs[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->GlobalRegularIds),
-      this->OutGlobalRegularIds);
+      this->OutData.GlobalRegularIds);
     // outDataValues[outAttachmentPoint]  =  dataValues[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->DataValues),
-      this->OutDataValues);
+      this->OutData.DataValues);
     // outSupernodeIDs[outAttachmentPoint]  =  supernodeIDs[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->SupernodeIds),
-      this->OutSupernodeIds);
+      this->OutData.SupernodeIds);
     // outSuperparents[outAttachmentPoint]  =  superparents[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->Superparents),
-      this->OutSuperparents);
+      this->OutData.Superparents);
     // outSuperparentRounds[outAttachmentPoint]  =  superparentRounds[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->SuperparentRounds),
-      this->OutSuperparentRounds);
+      this->OutData.SuperparentRounds);
     // outWhichRounds[outAttachmentPoint]  =  whichRounds[attachmentPoint];
     vtkm::cont::Algorithm::Copy(
       vtkm::cont::make_ArrayHandlePermutation(this->AttachmentIds, this->WhichRounds),
-      this->OutWhichRounds);
+      this->OutData.WhichRounds);
   }
 
   // clean up memory
@@ -399,13 +402,13 @@ void HierarchicalAugmenter<FieldType>::PrepareOutAttachmentPoints(vtkm::Id round
 
 // routine to add partner's current list of attachment points
 template <typename FieldType>
-void HierarchicalAugmenter<FieldType>::RetrieveInAttachmentPoints(HierarchicalAugmenter& partner)
+void HierarchicalAugmenter<FieldType>::RetrieveInAttachmentPoints()
 { // RetrieveInAttachmentPoints()
   // what we want to do here is to copy all of the partner's attachments for the round into our own buffer
   // this code will be replaced in the MPI with a suitable transmit / receive
   // store the current size
   vtkm::Id numAttachmentsCurrently = this->GlobalRegularIds.GetNumberOfValues();
-  vtkm::Id numIncomingAttachments = partner.OutGlobalRegularIds.GetNumberOfValues();
+  vtkm::Id numIncomingAttachments = InData.GlobalRegularIds.GetNumberOfValues();
   vtkm::Id numTotalAttachments = numAttachmentsCurrently + numIncomingAttachments;
 
   // I.  resize the existing arrays
@@ -419,48 +422,44 @@ void HierarchicalAugmenter<FieldType>::RetrieveInAttachmentPoints(HierarchicalAu
   // II. copy the additional points into them
   {
     // The following sequence of copy operations implements the following for from the orginal code
-    // for (vtkm::Id outAttachmentPoint = 0; outAttachmentPoint < partner.outGlobalRegularIDs.size(); outAttachmentPoint++)
+    // for (vtkm::Id inAttachmentPoint = 0; inAttachmentPoint < inGlobalRegularIDs.size(); inAttachmentPoint++)
-    // globalRegularIDs[attachmentPoint]  =  partner.outGlobalRegularIDs[outAttachmentPoint];
+    // globalRegularIDs[attachmentPoint]  =  inGlobalRegularIDs[outAttachmentPoint];
     auto tempGlobalRegularIdsView = vtkm::cont::make_ArrayHandleView(
       this->GlobalRegularIds, numAttachmentsCurrently, numIncomingAttachments);
-    vtkm::cont::Algorithm::Copy(partner.OutGlobalRegularIds, tempGlobalRegularIdsView);
+    vtkm::cont::Algorithm::Copy(InData.GlobalRegularIds, tempGlobalRegularIdsView);
-    // dataValues[attachmentPoint]  =  partner.outDataValues[outAttachmentPoint];
+    // dataValues[attachmentPoint]  =  inDataValues[outAttachmentPoint];
     auto tempDataValuesView = vtkm::cont::make_ArrayHandleView(
       this->DataValues, numAttachmentsCurrently, numIncomingAttachments);
-    vtkm::cont::Algorithm::Copy(partner.OutDataValues, tempDataValuesView);
+    vtkm::cont::Algorithm::Copy(InData.DataValues, tempDataValuesView);
     // supernodeIDs[attachmentPoint]  =  NO_SUCH_ELEMENT;
     auto tempNoSuchElementArr = vtkm::cont::make_ArrayHandleConstant(
       vtkm::worklet::contourtree_augmented::NO_SUCH_ELEMENT, numIncomingAttachments);
     auto tempSupernodeIdsView = vtkm::cont::make_ArrayHandleView(
       this->SupernodeIds, numAttachmentsCurrently, numIncomingAttachments);
     vtkm::cont::Algorithm::Copy(tempNoSuchElementArr, tempSupernodeIdsView);
-    // superparents[attachmentPoint]  =  partner.outSuperparents[outAttachmentPoint];
+    // superparents[attachmentPoint]  =  inSuperparents[outAttachmentPoint];
     auto tempSuperparentsView = vtkm::cont::make_ArrayHandleView(
       this->Superparents, numAttachmentsCurrently, numIncomingAttachments);
-    vtkm::cont::Algorithm::Copy(partner.OutSuperparents, tempSuperparentsView);
+    vtkm::cont::Algorithm::Copy(InData.Superparents, tempSuperparentsView);
-    // superparentRounds[attachmentPoint]  =  partner.outSuperparentRounds[outAttachmentPoint];
+    // superparentRounds[attachmentPoint]  =  inSuperparentRounds[outAttachmentPoint];
     auto tempSuperparentRoundsView = vtkm::cont::make_ArrayHandleView(
       this->SuperparentRounds, numAttachmentsCurrently, numIncomingAttachments);
-    vtkm::cont::Algorithm::Copy(partner.OutSuperparentRounds, tempSuperparentRoundsView);
+    vtkm::cont::Algorithm::Copy(InData.SuperparentRounds, tempSuperparentRoundsView);
-    // whichRounds[attachmentPoint]  =  partner.outWhichRounds[outAttachmentPoint];
+    // whichRounds[attachmentPoint]  =  inWhichRounds[outAttachmentPoint];
     auto tempWhichRoundsView = vtkm::cont::make_ArrayHandleView(
       this->WhichRounds, numAttachmentsCurrently, numIncomingAttachments);
-    vtkm::cont::Algorithm::Copy(partner.OutWhichRounds, tempWhichRoundsView);
+    vtkm::cont::Algorithm::Copy(InData.WhichRounds, tempWhichRoundsView);
   }
 } // RetrieveInAttachmentPoints()
 
 
 // routine to release memory used for out arrays
 template <typename FieldType>
-void HierarchicalAugmenter<FieldType>::ReleaseOutArrays()
+void HierarchicalAugmenter<FieldType>::ReleaseSwapArrays()
-{ // ReleaseOutArrays()
+{ // ReleaseSwapArrays()
-  this->OutGlobalRegularIds.ReleaseResources();
+  this->OutData.ReleaseResources();
-  this->OutDataValues.ReleaseResources();
+  this->InData.ReleaseResources();
-  this->OutSupernodeIds.ReleaseResources();
+} // ReleaseSwapArrays()
-  this->OutSuperparents.ReleaseResources();
-  this->OutSuperparentRounds.ReleaseResources();
-  this->OutWhichRounds.ReleaseResources();
-} // ReleaseOutArrays()
 
 
 // routine to reconstruct a hierarchical tree using the augmenting supernodes
@@ -1317,19 +1316,36 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
     "WhichRounds", this->WhichRounds, -1, resultStream);
   resultStream << std::endl;
   resultStream << "Outgoing Attachment Points" << std::endl;
-  vtkm::worklet::contourtree_augmented::PrintHeader(this->OutGlobalRegularIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintHeader(
+    this->OutData.GlobalRegularIds.GetNumberOfValues());
   vtkm::worklet::contourtree_augmented::PrintIndices(
-    "Out Global Regular Ids", this->OutGlobalRegularIds, -1, resultStream);
+    "Out Global Regular Ids", this->OutData.GlobalRegularIds, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintValues(
-    "Out Data Values", this->OutDataValues, -1, resultStream);
+    "Out Data Values", this->OutData.DataValues, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
-    "Out Supernode Ids", this->OutSupernodeIds, -1, resultStream);
+    "Out Supernode Ids", this->OutData.SupernodeIds, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
-    "Out Superparents", this->OutSuperparents, -1, resultStream);
+    "Out Superparents", this->OutData.Superparents, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
-    "Out Superparent Rounds", this->OutSuperparentRounds, -1, resultStream);
+    "Out Superparent Rounds", this->OutData.SuperparentRounds, -1, resultStream);
   vtkm::worklet::contourtree_augmented::PrintIndices(
-    "Out WhichRounds", this->OutWhichRounds, -1, resultStream);
+    "Out WhichRounds", this->OutData.WhichRounds, -1, resultStream);
+  resultStream << std::endl;
+  resultStream << "Incoming Attachment Points" << std::endl;
+  vtkm::worklet::contourtree_augmented::PrintHeader(
+    this->InData.GlobalRegularIds.GetNumberOfValues());
+  vtkm::worklet::contourtree_augmented::PrintIndices(
+    "In Global Regular Ids", this->InData.GlobalRegularIds, -1, resultStream);
+  vtkm::worklet::contourtree_augmented::PrintValues(
+    "In Data Values", this->InData.DataValues, -1, resultStream);
+  vtkm::worklet::contourtree_augmented::PrintIndices(
+    "In Supernode Ids", this->InData.SupernodeIds, -1, resultStream);
+  vtkm::worklet::contourtree_augmented::PrintIndices(
+    "In Superparents", this->InData.Superparents, -1, resultStream);
+  vtkm::worklet::contourtree_augmented::PrintIndices(
+    "In Superparent Rounds", this->InData.SuperparentRounds, -1, resultStream);
+  vtkm::worklet::contourtree_augmented::PrintIndices(
+    "In WhichRounds", this->InData.WhichRounds, -1, resultStream);
   resultStream << std::endl;
   resultStream << "Holding Arrays" << std::endl;
   vtkm::worklet::contourtree_augmented::PrintHeader(
@@ -1402,46 +1418,9 @@ std::string HierarchicalAugmenter<FieldType>::DebugPrint(std::string message,
 } // DebugPrint()
 
 
-
-
 } // namespace contourtree_distributed
 } // namespace worklet
 } // namespace vtkm
 
 
-namespace vtkmdiy
-{
-
-// Struct to serialize ContourTreeMesh objects (i.e., load/save) needed in parralle for DIY
-template <typename FieldType>
-struct Serialization<vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>>
-{
-  static void save(
-    vtkmdiy::BinaryBuffer& bb,
-    const vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>& ha)
-  {
-    vtkmdiy::save(bb, ha.OutGlobalRegularIds);
-    vtkmdiy::save(bb, ha.OutDataValues);
-    vtkmdiy::save(bb, ha.OutSupernodeIds);
-    vtkmdiy::save(bb, ha.OutSuperparents);
-    vtkmdiy::save(bb, ha.OutSuperparentRounds);
-    vtkmdiy::save(bb, ha.OutWhichRounds);
-  }
-
-  static void load(vtkmdiy::BinaryBuffer& bb,
-                   vtkm::worklet::contourtree_distributed::HierarchicalAugmenter<FieldType>& ha)
-  {
-    // TODO/FIXME: Save to Out or some other array? Shoud possibly InGlobalRegularIds etc.. Please check!
-    vtkmdiy::load(bb, ha.OutGlobalRegularIds);
-    vtkmdiy::load(bb, ha.OutDataValues);
-    vtkmdiy::load(bb, ha.OutSupernodeIds);
-    vtkmdiy::load(bb, ha.OutSuperparents);
-    vtkmdiy::load(bb, ha.OutSuperparentRounds);
-    vtkmdiy::load(bb, ha.OutWhichRounds);
-  }
-};
-
-} // namespace mangled_vtkmdiy_namespace
-
-
 #endif

vtkm/worklet/contourtree_distributed/hierarchical_augmenter/CMakeLists.txt

@@ -23,6 +23,7 @@ set(headers
   AttachmentAndSupernodeComparator.h
   ResizeArraysBuildNewSupernodeIdsWorklet.h
   CreateSuperarcsWorklet.h
+  HierarchicalAugmenterInOutData.h
 )
 
 vtkm_declare_headers(${headers})

vtkm/worklet/contourtree_distributed/hierarchical_augmenter/HierarchicalAugmenterInOutData.h

@@ -0,0 +1,167 @@
+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//============================================================================
+// Copyright (c) 2018, The Regents of the University of California, through
+// Lawrence Berkeley National Laboratory (subject to receipt of any required approvals
+// from the U.S. Dept. of Energy).  All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// (1) Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+//
+// (2) Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+// (3) Neither the name of the University of California, Lawrence Berkeley National
+//     Laboratory, U.S. Dept. of Energy nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//=======================================================================================
+//
+//  Parallel Peak Pruning v. 2.0
+//
+//  Started June 15, 2017
+//
+// Copyright Hamish Carr, University of Leeds
+//
+// HierarchicalAugmenter.h
+//
+//=======================================================================================
+
+#ifndef vtk_m_worklet_contourtree_distributed_hierarchical_augmenter_hierarchical_augmenter_in_out_data_h
+#define vtk_m_worklet_contourtree_distributed_hierarchical_augmenter_hierarchical_augmenter_in_out_data_h
+
+
+#include <vtkm/worklet/contourtree_augmented/Types.h>
+
+
+namespace vtkm
+{
+namespace worklet
+{
+namespace contourtree_distributed
+{
+namespace hierarchical_augmenter
+{
+
+/// Class for storing input or output data for the HierarchicalAugmenter. The  data is factored out in this class to
+/// allow for modular code and easue reuse, since the input and output require the same types of array parameters
+template <typename FieldType>
+class HierarchicalAugmenterInOutData
+{ // class HierarchicalAugmenter
+public:
+  vtkm::worklet::contourtree_augmented::IdArrayType GlobalRegularIds;
+  vtkm::cont::ArrayHandle<FieldType> DataValues;
+  vtkm::worklet::contourtree_augmented::IdArrayType SupernodeIds;
+  vtkm::worklet::contourtree_augmented::IdArrayType Superparents;
+  vtkm::worklet::contourtree_augmented::IdArrayType SuperparentRounds;
+  vtkm::worklet::contourtree_augmented::IdArrayType WhichRounds;
+
+  /// empty constructor
+  HierarchicalAugmenterInOutData() {}
+
+  /// main constructor
+  HierarchicalAugmenterInOutData(
+    vtkm::worklet::contourtree_augmented::IdArrayType& globalRegularIds,
+    vtkm::cont::ArrayHandle<FieldType>& dataValues,
+    vtkm::worklet::contourtree_augmented::IdArrayType& supernodeIds,
+    vtkm::worklet::contourtree_augmented::IdArrayType& superparents,
+    vtkm::worklet::contourtree_augmented::IdArrayType& superparentRounds,
+    vtkm::worklet::contourtree_augmented::IdArrayType& whichRounds)
+    : GlobalRegularIds(globalRegularIds)
+    , DataValues(dataValues)
+    , SupernodeIds(supernodeIds)
+    , Superparents(superparents)
+    , SuperparentRounds(superparentRounds)
+    , WhichRounds(whichRounds)
+  {
+  }
+
+  /// Destructor
+  ~HierarchicalAugmenterInOutData();
+
+  /// Clear all arrays
+  void ReleaseResources();
+
+}; // class HierarchicalAugmenterInOutData
+
+template <typename FieldType>
+HierarchicalAugmenterInOutData<FieldType>::~HierarchicalAugmenterInOutData()
+{
+  this->ReleaseResources();
+}
+
+// routine to release memory used for out arrays
+template <typename FieldType>
+void HierarchicalAugmenterInOutData<FieldType>::ReleaseResources()
+{ // ReleaseResources()
+  this->GlobalRegularIds.ReleaseResources();
+  this->DataValues.ReleaseResources();
+  this->SupernodeIds.ReleaseResources();
+  this->Superparents.ReleaseResources();
+  this->SuperparentRounds.ReleaseResources();
+  this->WhichRounds.ReleaseResources();
+} // ReleaseResources()
+
+} // namespace hierarchical_augmenter
+} // namespace contourtree_distributed
+} // namespace worklet
+} // namespace vtkm
+
+namespace vtkmdiy
+{
+
+// Struct to serialize ContourTreeMesh objects (i.e., load/save) needed in parralle for DIY
+template <typename FieldType>
+struct Serialization<vtkm::worklet::contourtree_distributed::hierarchical_augmenter::
+                       HierarchicalAugmenterInOutData<FieldType>>
+{
+  static void save(vtkmdiy::BinaryBuffer& bb,
+                   const vtkm::worklet::contourtree_distributed::hierarchical_augmenter::
+                     HierarchicalAugmenterInOutData<FieldType>& ha)
+  {
+    vtkmdiy::save(bb, ha.GlobalRegularIds);
+    vtkmdiy::save(bb, ha.DataValues);
+    vtkmdiy::save(bb, ha.SupernodeIds);
+    vtkmdiy::save(bb, ha.Superparents);
+    vtkmdiy::save(bb, ha.SuperparentRounds);
+    vtkmdiy::save(bb, ha.WhichRounds);
+  }
+
+  static void load(vtkmdiy::BinaryBuffer& bb,
+                   vtkm::worklet::contourtree_distributed::hierarchical_augmenter::
+                     HierarchicalAugmenterInOutData<FieldType>& ha)
+  {
+    vtkmdiy::load(bb, ha.GlobalRegularIds);
+    vtkmdiy::load(bb, ha.DataValues);
+    vtkmdiy::load(bb, ha.SupernodeIds);
+    vtkmdiy::load(bb, ha.Superparents);
+    vtkmdiy::load(bb, ha.SuperparentRounds);
+    vtkmdiy::load(bb, ha.WhichRounds);
+  }
+};
+
+} // namespace mangled_vtkmdiy_namespace
+
+#endif