From 2af555f6c9923d1e6fb29f91258e9641d3039b0c Mon Sep 17 00:00:00 2001 From: Kenneth Moreland Date: Thu, 2 Mar 2023 15:43:23 -0700 Subject: [PATCH] Simplify serialization of DataSet objects `vtkm::cont::DataSet` is a dynamic object that can hold cell sets and fields of many different types, none of which are known until runtime. This causes a problem with serialization, which has to know what type to compile the serialization for, particularly when unserializing the type at the receiving end. The original implementation "solved" the problem by creating a secondary wrapper object that was templated on types of field arrays and cell sets that might be serialized. This is not a great solution as it punts the problem to algorithm developers. This problem has been completely solved for fields, as it is possible to serialize most types of arrays without knowing their type now. You still need to iterate over every possible `CellSet` type, but there are not that many `CellSet`s that are practically encountered. Thus, there is now a direct implementation of `Serialization` for `DataSet` that covers all the data types you are likely to encounter. The old `SerializableDataSet` has been deprecated. In the unlikely event an algorithm needs to transfer a non-standard type of `CellSet` (such as a permuted cell set), it can use the replacement `DataSetWithCellSetTypes`, which just specifies the cell set types. --- docs/changelog/serialize-dataset.md | 22 ++++++ .../RedistributePoints.cxx | 10 ++- vtkm/cont/DataSet.cxx | 31 ++++++++ vtkm/cont/DataSet.h | 71 +++++++++++++++---- .../testing/UnitTestSerializationDataSet.cxx | 16 ++++- ...stingContourTreeUniformDistributedFilter.h | 12 ++-- 6 files changed, 133 insertions(+), 29 deletions(-) create mode 100644 docs/changelog/serialize-dataset.md diff --git a/docs/changelog/serialize-dataset.md b/docs/changelog/serialize-dataset.md new file mode 100644 index 000000000..a00831ac2 --- /dev/null +++ b/docs/changelog/serialize-dataset.md @@ -0,0 +1,22 @@ +# Simplified serialization of DataSet objects + +`vtkm::cont::DataSet` is a dynamic object that can hold cell sets and +fields of many different types, none of which are known until runtime. This +causes a problem with serialization, which has to know what type to compile +the serialization for, particularly when unserializing the type at the +receiving end. The original implementation "solved" the problem by creating +a secondary wrapper object that was templated on types of field arrays and +cell sets that might be serialized. This is not a great solution as it +punts the problem to algorithm developers. + +This problem has been completely solved for fields, as it is possible to +serialize most types of arrays without knowing their type now. You still +need to iterate over every possible `CellSet` type, but there are not that +many `CellSet`s that are practically encountered. Thus, there is now a +direct implementation of `Serialization` for `DataSet` that covers all the +data types you are likely to encounter. + +The old `SerializableDataSet` has been deprecated. In the unlikely event an +algorithm needs to transfer a non-standard type of `CellSet` (such as a +permuted cell set), it can use the replacement `DataSetWithCellSetTypes`, +which just specifies the cell set types. diff --git a/examples/redistribute_points/RedistributePoints.cxx b/examples/redistribute_points/RedistributePoints.cxx index 33e6f3084..4f5fdf32f 100644 --- a/examples/redistribute_points/RedistributePoints.cxx +++ b/examples/redistribute_points/RedistributePoints.cxx @@ -133,8 +133,7 @@ public: this->Decomposer.fill_bounds(bds, target.gid); auto extractedDS = this->Extract(*block, bds); - // TODO: Need a better way to serialize DataSet. See issue #725. - rp.enqueue(target, vtkm::cont::SerializableDataSet<>(extractedDS)); + rp.enqueue(target, extractedDS); } // clear our dataset. *block = vtkm::cont::DataSet(); @@ -149,10 +148,9 @@ public: auto target = rp.in_link().target(cc); if (rp.incoming(target.gid).size() > 0) { - // TODO: Need a better way to serialize DataSet. See issue #725. - vtkm::cont::SerializableDataSet<> sds; - rp.dequeue(target.gid, sds); - receives.push_back(sds.DataSet); + vtkm::cont::DataSet incomingDS; + rp.dequeue(target.gid, incomingDS); + receives.push_back(incomingDS); numValues += receives.back().GetCoordinateSystem(0).GetNumberOfPoints(); } } diff --git a/vtkm/cont/DataSet.cxx b/vtkm/cont/DataSet.cxx index 34c5b63bf..608de4fc8 100644 --- a/vtkm/cont/DataSet.cxx +++ b/vtkm/cont/DataSet.cxx @@ -8,6 +8,10 @@ // PURPOSE. See the above copyright notice for more information. //============================================================================ +#include +#include +#include +#include #include #include #include @@ -341,3 +345,30 @@ void DataSet::ConvertToExpected() } // namespace cont } // namespace vtkm + + +namespace mangled_diy_namespace +{ + +using SerializedCellSetTypes = vtkm::ListAppend, + vtkm::cont::CellSetStructured<2>, + vtkm::cont::CellSetStructured<3>, + vtkm::cont::CellSetExplicit<>, + vtkm::cont::CellSetSingleType<>, + vtkm::cont::CellSetExtrude>>; +using DefaultDataSetWithCellTypes = vtkm::cont::DataSetWithCellSetTypes; + +void Serialization::save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj) +{ + vtkmdiy::save(bb, DefaultDataSetWithCellTypes{ obj }); +} + +void Serialization::load(BinaryBuffer& bb, vtkm::cont::DataSet& obj) +{ + DefaultDataSetWithCellTypes data; + vtkmdiy::load(bb, data); + obj = data.DataSet; +} + +} // namespace mangled_diy_namespace diff --git a/vtkm/cont/DataSet.h b/vtkm/cont/DataSet.h index 2cef4af8d..c55a193b7 100644 --- a/vtkm/cont/DataSet.h +++ b/vtkm/cont/DataSet.h @@ -420,37 +420,70 @@ namespace vtkm namespace cont { -template -struct SerializableDataSet +/// \brief Specify cell sets to use when serializing a `DataSet`. +/// +/// Usually when serializing a `DataSet`, it uses a fixed set of standard +/// `CellSet` types to serialize. If you are writing an algorithm with a +/// custom `CellSet`, you can specify the `CellSet`(s) as the template +/// parameter for this class (either as a list of `CellSet`s or in a +/// single `vtkm::List` parameter). +/// +template +struct DataSetWithCellSetTypes { - SerializableDataSet() = default; + vtkm::cont::DataSet DataSet; - explicit SerializableDataSet(const vtkm::cont::DataSet& dataset) + DataSetWithCellSetTypes() = default; + + explicit DataSetWithCellSetTypes(const vtkm::cont::DataSet& dataset) : DataSet(dataset) { } - - vtkm::cont::DataSet DataSet; }; + +template +struct DataSetWithCellSetTypes> + : DataSetWithCellSetTypes +{ + using DataSetWithCellSetTypes::DataSetWithCellSetTypes; +}; + +template +struct VTKM_DEPRECATED( + 2.1, + "Serialize DataSet directly or use DataSetWithCellSetTypes for weird CellSets.") + SerializableDataSet : DataSetWithCellSetTypes +{ + using DataSetWithCellSetTypes::DataSetWithCellSetTypes; +}; + } } // vtkm::cont namespace mangled_diy_namespace { -template -struct Serialization> +template <> +struct VTKM_CONT_EXPORT Serialization +{ + static VTKM_CONT void foo(); + static VTKM_CONT void save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj); + static VTKM_CONT void load(BinaryBuffer& bb, vtkm::cont::DataSet& obj); +}; + +template +struct Serialization> { private: - using Type = vtkm::cont::SerializableDataSet; + using Type = vtkm::cont::DataSetWithCellSetTypes; public: static VTKM_CONT void save(BinaryBuffer& bb, const Type& serializable) { const auto& dataset = serializable.DataSet; - vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(CellSetTypesList{})); + vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(vtkm::List{})); vtkm::IdComponent numberOfFields = dataset.GetNumberOfFields(); vtkmdiy::save(bb, numberOfFields); @@ -472,7 +505,7 @@ public: auto& dataset = serializable.DataSet; dataset = {}; // clear - vtkm::cont::UncertainCellSet cells; + vtkm::cont::UncertainCellSet> cells; vtkmdiy::load(bb, cells); dataset.SetCellSet(cells); @@ -496,6 +529,20 @@ public: } }; +template +struct Serialization>> + : Serialization> +{ +}; + +VTKM_DEPRECATED_SUPPRESS_BEGIN +template +struct Serialization> + : Serialization> +{ +}; +VTKM_DEPRECATED_SUPPRESS_END + } // diy /// @endcond SERIALIZATION diff --git a/vtkm/cont/testing/UnitTestSerializationDataSet.cxx b/vtkm/cont/testing/UnitTestSerializationDataSet.cxx index 4299688d6..0c8af712d 100644 --- a/vtkm/cont/testing/UnitTestSerializationDataSet.cxx +++ b/vtkm/cont/testing/UnitTestSerializationDataSet.cxx @@ -22,16 +22,26 @@ using CellSetTypes = vtkm::List, vtkm::cont::CellSetStructured<2>, vtkm::cont::CellSetStructured<3>>; -using DataSetWrapper = vtkm::cont::SerializableDataSet; +using DataSetWrapper = vtkm::cont::DataSetWithCellSetTypes; -VTKM_CONT void TestEqualDataSet(const DataSetWrapper& ds1, const DataSetWrapper& ds2) +VTKM_CONT void TestEqualDataSetWrapper(const DataSetWrapper& ds1, const DataSetWrapper& ds2) { VTKM_TEST_ASSERT(test_equal_DataSets(ds1.DataSet, ds2.DataSet, CellSetTypes{})); } +VTKM_CONT void TestEqualDataSet(const vtkm::cont::DataSet& ds1, const vtkm::cont::DataSet& ds2) +{ + VTKM_TEST_ASSERT(test_equal_DataSets(ds1, ds2, CellSetTypes{})); +} + void RunTest(const vtkm::cont::DataSet& ds) { - TestSerialization(DataSetWrapper(ds), TestEqualDataSet); + VTKM_DEPRECATED_SUPPRESS_BEGIN + TestSerialization(vtkm::cont::SerializableDataSet(ds), + TestEqualDataSetWrapper); + VTKM_DEPRECATED_SUPPRESS_END + TestSerialization(DataSetWrapper(ds), TestEqualDataSetWrapper); + TestSerialization(ds, TestEqualDataSet); } void TestDataSetSerialization() diff --git a/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h b/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h index c511aafb0..a43db23a3 100644 --- a/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h +++ b/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h @@ -314,9 +314,6 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( { // Mutiple ranks -> Some assembly required. Collect data // on rank 0, all other ranks return empty data sets - using FieldTypeList = vtkm::ListAppend>; - using DataSetWrapper = - vtkm::cont::SerializableDataSet; // Communicate results to rank 0 auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator(); @@ -331,8 +328,7 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( p.enqueue(root, result.GetNumberOfPartitions()); for (const vtkm::cont::DataSet& curr_ds : result) { - auto curr_sds = DataSetWrapper(curr_ds); - p.enqueue(root, curr_sds); + p.enqueue(root, curr_ds); } }); // Exchange data, i.e., send to rank 0 (pass "true" to exchange data between @@ -352,9 +348,9 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( for (vtkm::Id currReceiveDataSetNo = 0; currReceiveDataSetNo < numberOfDataSetsToReceive; ++currReceiveDataSetNo) { - vtkm::cont::SerializableDataSet<> sds; - p.dequeue({ receiveFromRank, receiveFromRank }, sds); - combined_result.AppendPartition(sds.DataSet); + vtkm::cont::DataSet dsIncoming; + p.dequeue({ receiveFromRank, receiveFromRank }, dsIncoming); + combined_result.AppendPartition(dsIncoming); } } });