Simplify serialization of DataSet objects

`vtkm::cont::DataSet` is a dynamic object that can hold cell sets and
fields of many different types, none of which are known until runtime. This
causes a problem with serialization, which has to know what type to compile
the serialization for, particularly when unserializing the type at the
receiving end. The original implementation "solved" the problem by creating
a secondary wrapper object that was templated on types of field arrays and
cell sets that might be serialized. This is not a great solution as it
punts the problem to algorithm developers.

This problem has been completely solved for fields, as it is possible to
serialize most types of arrays without knowing their type now. You still
need to iterate over every possible `CellSet` type, but there are not that
many `CellSet`s that are practically encountered. Thus, there is now a
direct implementation of `Serialization` for `DataSet` that covers all the
data types you are likely to encounter.

The old `SerializableDataSet` has been deprecated. In the unlikely event an
algorithm needs to transfer a non-standard type of `CellSet` (such as a
permuted cell set), it can use the replacement `DataSetWithCellSetTypes`,
which just specifies the cell set types.

docs/changelog/serialize-dataset.md

@@ -0,0 +1,22 @@
+# Simplified serialization of DataSet objects
+
+`vtkm::cont::DataSet` is a dynamic object that can hold cell sets and
+fields of many different types, none of which are known until runtime. This
+causes a problem with serialization, which has to know what type to compile
+the serialization for, particularly when unserializing the type at the
+receiving end. The original implementation "solved" the problem by creating
+a secondary wrapper object that was templated on types of field arrays and
+cell sets that might be serialized. This is not a great solution as it
+punts the problem to algorithm developers.
+
+This problem has been completely solved for fields, as it is possible to
+serialize most types of arrays without knowing their type now. You still
+need to iterate over every possible `CellSet` type, but there are not that
+many `CellSet`s that are practically encountered. Thus, there is now a
+direct implementation of `Serialization` for `DataSet` that covers all the
+data types you are likely to encounter.
+
+The old `SerializableDataSet` has been deprecated. In the unlikely event an
+algorithm needs to transfer a non-standard type of `CellSet` (such as a
+permuted cell set), it can use the replacement `DataSetWithCellSetTypes`,
+which just specifies the cell set types.

examples/redistribute_points/RedistributePoints.cxx

@@ -133,8 +133,7 @@ public:
           this->Decomposer.fill_bounds(bds, target.gid);
 
           auto extractedDS = this->Extract(*block, bds);
-          // TODO: Need a better way to serialize DataSet. See issue #725.
-          rp.enqueue(target, vtkm::cont::SerializableDataSet<>(extractedDS));
+          rp.enqueue(target, extractedDS);
         }
         // clear our dataset.
         *block = vtkm::cont::DataSet();
@@ -149,10 +148,9 @@ public:
         auto target = rp.in_link().target(cc);
         if (rp.incoming(target.gid).size() > 0)
         {
-          // TODO: Need a better way to serialize DataSet. See issue #725.
-          vtkm::cont::SerializableDataSet<> sds;
-          rp.dequeue(target.gid, sds);
-          receives.push_back(sds.DataSet);
+          vtkm::cont::DataSet incomingDS;
+          rp.dequeue(target.gid, incomingDS);
+          receives.push_back(incomingDS);
           numValues += receives.back().GetCoordinateSystem(0).GetNumberOfPoints();
         }
       }

vtkm/cont/DataSet.cxx

@@ -8,6 +8,10 @@
 //  PURPOSE.  See the above copyright notice for more information.
 //============================================================================
 
+#include <vtkm/cont/CellSetExplicit.h>
+#include <vtkm/cont/CellSetExtrude.h>
+#include <vtkm/cont/CellSetSingleType.h>
+#include <vtkm/cont/CellSetStructured.h>
 #include <vtkm/cont/DataSet.h>
 #include <vtkm/cont/ErrorBadValue.h>
 #include <vtkm/cont/Logging.h>
@@ -341,3 +345,30 @@ void DataSet::ConvertToExpected()
 
 } // namespace cont
 } // namespace vtkm
+
+
+namespace mangled_diy_namespace
+{
+
+using SerializedCellSetTypes = vtkm::ListAppend<VTKM_DEFAULT_CELL_SET_LIST,
+                                                vtkm::List<vtkm::cont::CellSetStructured<1>,
+                                                           vtkm::cont::CellSetStructured<2>,
+                                                           vtkm::cont::CellSetStructured<3>,
+                                                           vtkm::cont::CellSetExplicit<>,
+                                                           vtkm::cont::CellSetSingleType<>,
+                                                           vtkm::cont::CellSetExtrude>>;
+using DefaultDataSetWithCellTypes = vtkm::cont::DataSetWithCellSetTypes<SerializedCellSetTypes>;
+
+void Serialization<vtkm::cont::DataSet>::save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj)
+{
+  vtkmdiy::save(bb, DefaultDataSetWithCellTypes{ obj });
+}
+
+void Serialization<vtkm::cont::DataSet>::load(BinaryBuffer& bb, vtkm::cont::DataSet& obj)
+{
+  DefaultDataSetWithCellTypes data;
+  vtkmdiy::load(bb, data);
+  obj = data.DataSet;
+}
+
+} // namespace mangled_diy_namespace

vtkm/cont/DataSet.h

@@ -420,37 +420,70 @@ namespace vtkm
 namespace cont
 {
 
-template <typename FieldTypeList = VTKM_DEFAULT_TYPE_LIST,
-          typename CellSetTypesList = VTKM_DEFAULT_CELL_SET_LIST>
-struct SerializableDataSet
+/// \brief Specify cell sets to use when serializing a `DataSet`.
+///
+/// Usually when serializing a `DataSet`, it uses a fixed set of standard
+/// `CellSet` types to serialize. If you are writing an algorithm with a
+/// custom `CellSet`, you can specify the `CellSet`(s) as the template
+/// parameter for this class (either as a list of `CellSet`s or in a
+/// single `vtkm::List` parameter).
+///
+template <typename... CellSetTypes>
+struct DataSetWithCellSetTypes
 {
-  SerializableDataSet() = default;
+  vtkm::cont::DataSet DataSet;
 
-  explicit SerializableDataSet(const vtkm::cont::DataSet& dataset)
+  DataSetWithCellSetTypes() = default;
+
+  explicit DataSetWithCellSetTypes(const vtkm::cont::DataSet& dataset)
     : DataSet(dataset)
   {
   }
-
-  vtkm::cont::DataSet DataSet;
 };
+
+template <typename... CellSetTypes>
+struct DataSetWithCellSetTypes<vtkm::List<CellSetTypes...>>
+  : DataSetWithCellSetTypes<CellSetTypes...>
+{
+  using DataSetWithCellSetTypes<CellSetTypes...>::DataSetWithCellSetTypes;
+};
+
+template <typename FieldTypeList = VTKM_DEFAULT_TYPE_LIST,
+          typename CellSetTypesList = VTKM_DEFAULT_CELL_SET_LIST>
+struct VTKM_DEPRECATED(
+  2.1,
+  "Serialize DataSet directly or use DataSetWithCellSetTypes for weird CellSets.")
+  SerializableDataSet : DataSetWithCellSetTypes<CellSetTypesList>
+{
+  using DataSetWithCellSetTypes<CellSetTypesList>::DataSetWithCellSetTypes;
+};
+
 }
 } // vtkm::cont
 
 namespace mangled_diy_namespace
 {
 
-template <typename FieldTypeList, typename CellSetTypesList>
-struct Serialization<vtkm::cont::SerializableDataSet<FieldTypeList, CellSetTypesList>>
+template <>
+struct VTKM_CONT_EXPORT Serialization<vtkm::cont::DataSet>
+{
+  static VTKM_CONT void foo();
+  static VTKM_CONT void save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj);
+  static VTKM_CONT void load(BinaryBuffer& bb, vtkm::cont::DataSet& obj);
+};
+
+template <typename... CellSetTypes>
+struct Serialization<vtkm::cont::DataSetWithCellSetTypes<CellSetTypes...>>
 {
 private:
-  using Type = vtkm::cont::SerializableDataSet<FieldTypeList, CellSetTypesList>;
+  using Type = vtkm::cont::DataSetWithCellSetTypes<CellSetTypes...>;
 
 public:
   static VTKM_CONT void save(BinaryBuffer& bb, const Type& serializable)
   {
     const auto& dataset = serializable.DataSet;
 
-    vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(CellSetTypesList{}));
+    vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(vtkm::List<CellSetTypes...>{}));
 
     vtkm::IdComponent numberOfFields = dataset.GetNumberOfFields();
     vtkmdiy::save(bb, numberOfFields);
@@ -472,7 +505,7 @@ public:
     auto& dataset = serializable.DataSet;
     dataset = {}; // clear
 
-    vtkm::cont::UncertainCellSet<CellSetTypesList> cells;
+    vtkm::cont::UncertainCellSet<vtkm::List<CellSetTypes...>> cells;
     vtkmdiy::load(bb, cells);
     dataset.SetCellSet(cells);
 
@@ -496,6 +529,20 @@ public:
   }
 };
 
+template <typename... CellSetTypes>
+struct Serialization<vtkm::cont::DataSetWithCellSetTypes<vtkm::List<CellSetTypes...>>>
+  : Serialization<vtkm::cont::DataSetWithCellSetTypes<CellSetTypes...>>
+{
+};
+
+VTKM_DEPRECATED_SUPPRESS_BEGIN
+template <typename FieldTypeList, typename CellSetTypesList>
+struct Serialization<vtkm::cont::SerializableDataSet<FieldTypeList, CellSetTypesList>>
+  : Serialization<vtkm::cont::DataSetWithCellSetTypes<CellSetTypesList>>
+{
+};
+VTKM_DEPRECATED_SUPPRESS_END
+
 } // diy
 /// @endcond SERIALIZATION
 

vtkm/cont/testing/UnitTestSerializationDataSet.cxx

@@ -22,16 +22,26 @@ using CellSetTypes = vtkm::List<vtkm::cont::CellSetExplicit<>,
                                 vtkm::cont::CellSetStructured<2>,
                                 vtkm::cont::CellSetStructured<3>>;
 
-using DataSetWrapper = vtkm::cont::SerializableDataSet<FieldTypeList, CellSetTypes>;
+using DataSetWrapper = vtkm::cont::DataSetWithCellSetTypes<CellSetTypes>;
 
-VTKM_CONT void TestEqualDataSet(const DataSetWrapper& ds1, const DataSetWrapper& ds2)
+VTKM_CONT void TestEqualDataSetWrapper(const DataSetWrapper& ds1, const DataSetWrapper& ds2)
 {
   VTKM_TEST_ASSERT(test_equal_DataSets(ds1.DataSet, ds2.DataSet, CellSetTypes{}));
 }
 
+VTKM_CONT void TestEqualDataSet(const vtkm::cont::DataSet& ds1, const vtkm::cont::DataSet& ds2)
+{
+  VTKM_TEST_ASSERT(test_equal_DataSets(ds1, ds2, CellSetTypes{}));
+}
+
 void RunTest(const vtkm::cont::DataSet& ds)
 {
-  TestSerialization(DataSetWrapper(ds), TestEqualDataSet);
+  VTKM_DEPRECATED_SUPPRESS_BEGIN
+  TestSerialization(vtkm::cont::SerializableDataSet<FieldTypeList, CellSetTypes>(ds),
+                    TestEqualDataSetWrapper);
+  VTKM_DEPRECATED_SUPPRESS_END
+  TestSerialization(DataSetWrapper(ds), TestEqualDataSetWrapper);
+  TestSerialization(ds, TestEqualDataSet);
 }
 
 void TestDataSetSerialization()

vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h

@@ -314,9 +314,6 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed(
   {
     // Mutiple ranks -> Some assembly required. Collect data
     // on rank 0, all other ranks return empty data sets
-    using FieldTypeList = vtkm::ListAppend<vtkm::TypeListScalarAll, vtkm::List<vtkm::Id>>;
-    using DataSetWrapper =
-      vtkm::cont::SerializableDataSet<FieldTypeList, vtkm::cont::CellSetListStructured>;
 
     // Communicate results to rank 0
     auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator();
@@ -331,8 +328,7 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed(
       p.enqueue(root, result.GetNumberOfPartitions());
       for (const vtkm::cont::DataSet& curr_ds : result)
       {
-        auto curr_sds = DataSetWrapper(curr_ds);
-        p.enqueue(root, curr_sds);
+        p.enqueue(root, curr_ds);
       }
     });
     // Exchange data, i.e., send to rank 0 (pass "true" to exchange data between
@@ -352,9 +348,9 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed(
           for (vtkm::Id currReceiveDataSetNo = 0; currReceiveDataSetNo < numberOfDataSetsToReceive;
                ++currReceiveDataSetNo)
           {
-            vtkm::cont::SerializableDataSet<> sds;
-            p.dequeue({ receiveFromRank, receiveFromRank }, sds);
-            combined_result.AppendPartition(sds.DataSet);
+            vtkm::cont::DataSet dsIncoming;
+            p.dequeue({ receiveFromRank, receiveFromRank }, dsIncoming);
+            combined_result.AppendPartition(dsIncoming);
           }
         }
       });