diff --git a/docs/changelog/serialize-dataset.md b/docs/changelog/serialize-dataset.md new file mode 100644 index 000000000..a00831ac2 --- /dev/null +++ b/docs/changelog/serialize-dataset.md @@ -0,0 +1,22 @@ +# Simplified serialization of DataSet objects + +`vtkm::cont::DataSet` is a dynamic object that can hold cell sets and +fields of many different types, none of which are known until runtime. This +causes a problem with serialization, which has to know what type to compile +the serialization for, particularly when unserializing the type at the +receiving end. The original implementation "solved" the problem by creating +a secondary wrapper object that was templated on types of field arrays and +cell sets that might be serialized. This is not a great solution as it +punts the problem to algorithm developers. + +This problem has been completely solved for fields, as it is possible to +serialize most types of arrays without knowing their type now. You still +need to iterate over every possible `CellSet` type, but there are not that +many `CellSet`s that are practically encountered. Thus, there is now a +direct implementation of `Serialization` for `DataSet` that covers all the +data types you are likely to encounter. + +The old `SerializableDataSet` has been deprecated. In the unlikely event an +algorithm needs to transfer a non-standard type of `CellSet` (such as a +permuted cell set), it can use the replacement `DataSetWithCellSetTypes`, +which just specifies the cell set types. diff --git a/examples/redistribute_points/RedistributePoints.cxx b/examples/redistribute_points/RedistributePoints.cxx index 33e6f3084..4f5fdf32f 100644 --- a/examples/redistribute_points/RedistributePoints.cxx +++ b/examples/redistribute_points/RedistributePoints.cxx @@ -133,8 +133,7 @@ public: this->Decomposer.fill_bounds(bds, target.gid); auto extractedDS = this->Extract(*block, bds); - // TODO: Need a better way to serialize DataSet. See issue #725. - rp.enqueue(target, vtkm::cont::SerializableDataSet<>(extractedDS)); + rp.enqueue(target, extractedDS); } // clear our dataset. *block = vtkm::cont::DataSet(); @@ -149,10 +148,9 @@ public: auto target = rp.in_link().target(cc); if (rp.incoming(target.gid).size() > 0) { - // TODO: Need a better way to serialize DataSet. See issue #725. - vtkm::cont::SerializableDataSet<> sds; - rp.dequeue(target.gid, sds); - receives.push_back(sds.DataSet); + vtkm::cont::DataSet incomingDS; + rp.dequeue(target.gid, incomingDS); + receives.push_back(incomingDS); numValues += receives.back().GetCoordinateSystem(0).GetNumberOfPoints(); } } diff --git a/vtkm/cont/DataSet.cxx b/vtkm/cont/DataSet.cxx index 34c5b63bf..608de4fc8 100644 --- a/vtkm/cont/DataSet.cxx +++ b/vtkm/cont/DataSet.cxx @@ -8,6 +8,10 @@ // PURPOSE. See the above copyright notice for more information. //============================================================================ +#include +#include +#include +#include #include #include #include @@ -341,3 +345,30 @@ void DataSet::ConvertToExpected() } // namespace cont } // namespace vtkm + + +namespace mangled_diy_namespace +{ + +using SerializedCellSetTypes = vtkm::ListAppend, + vtkm::cont::CellSetStructured<2>, + vtkm::cont::CellSetStructured<3>, + vtkm::cont::CellSetExplicit<>, + vtkm::cont::CellSetSingleType<>, + vtkm::cont::CellSetExtrude>>; +using DefaultDataSetWithCellTypes = vtkm::cont::DataSetWithCellSetTypes; + +void Serialization::save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj) +{ + vtkmdiy::save(bb, DefaultDataSetWithCellTypes{ obj }); +} + +void Serialization::load(BinaryBuffer& bb, vtkm::cont::DataSet& obj) +{ + DefaultDataSetWithCellTypes data; + vtkmdiy::load(bb, data); + obj = data.DataSet; +} + +} // namespace mangled_diy_namespace diff --git a/vtkm/cont/DataSet.h b/vtkm/cont/DataSet.h index 2cef4af8d..c55a193b7 100644 --- a/vtkm/cont/DataSet.h +++ b/vtkm/cont/DataSet.h @@ -420,37 +420,70 @@ namespace vtkm namespace cont { -template -struct SerializableDataSet +/// \brief Specify cell sets to use when serializing a `DataSet`. +/// +/// Usually when serializing a `DataSet`, it uses a fixed set of standard +/// `CellSet` types to serialize. If you are writing an algorithm with a +/// custom `CellSet`, you can specify the `CellSet`(s) as the template +/// parameter for this class (either as a list of `CellSet`s or in a +/// single `vtkm::List` parameter). +/// +template +struct DataSetWithCellSetTypes { - SerializableDataSet() = default; + vtkm::cont::DataSet DataSet; - explicit SerializableDataSet(const vtkm::cont::DataSet& dataset) + DataSetWithCellSetTypes() = default; + + explicit DataSetWithCellSetTypes(const vtkm::cont::DataSet& dataset) : DataSet(dataset) { } - - vtkm::cont::DataSet DataSet; }; + +template +struct DataSetWithCellSetTypes> + : DataSetWithCellSetTypes +{ + using DataSetWithCellSetTypes::DataSetWithCellSetTypes; +}; + +template +struct VTKM_DEPRECATED( + 2.1, + "Serialize DataSet directly or use DataSetWithCellSetTypes for weird CellSets.") + SerializableDataSet : DataSetWithCellSetTypes +{ + using DataSetWithCellSetTypes::DataSetWithCellSetTypes; +}; + } } // vtkm::cont namespace mangled_diy_namespace { -template -struct Serialization> +template <> +struct VTKM_CONT_EXPORT Serialization +{ + static VTKM_CONT void foo(); + static VTKM_CONT void save(BinaryBuffer& bb, const vtkm::cont::DataSet& obj); + static VTKM_CONT void load(BinaryBuffer& bb, vtkm::cont::DataSet& obj); +}; + +template +struct Serialization> { private: - using Type = vtkm::cont::SerializableDataSet; + using Type = vtkm::cont::DataSetWithCellSetTypes; public: static VTKM_CONT void save(BinaryBuffer& bb, const Type& serializable) { const auto& dataset = serializable.DataSet; - vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(CellSetTypesList{})); + vtkmdiy::save(bb, dataset.GetCellSet().ResetCellSetList(vtkm::List{})); vtkm::IdComponent numberOfFields = dataset.GetNumberOfFields(); vtkmdiy::save(bb, numberOfFields); @@ -472,7 +505,7 @@ public: auto& dataset = serializable.DataSet; dataset = {}; // clear - vtkm::cont::UncertainCellSet cells; + vtkm::cont::UncertainCellSet> cells; vtkmdiy::load(bb, cells); dataset.SetCellSet(cells); @@ -496,6 +529,20 @@ public: } }; +template +struct Serialization>> + : Serialization> +{ +}; + +VTKM_DEPRECATED_SUPPRESS_BEGIN +template +struct Serialization> + : Serialization> +{ +}; +VTKM_DEPRECATED_SUPPRESS_END + } // diy /// @endcond SERIALIZATION diff --git a/vtkm/cont/testing/UnitTestSerializationDataSet.cxx b/vtkm/cont/testing/UnitTestSerializationDataSet.cxx index 4299688d6..0c8af712d 100644 --- a/vtkm/cont/testing/UnitTestSerializationDataSet.cxx +++ b/vtkm/cont/testing/UnitTestSerializationDataSet.cxx @@ -22,16 +22,26 @@ using CellSetTypes = vtkm::List, vtkm::cont::CellSetStructured<2>, vtkm::cont::CellSetStructured<3>>; -using DataSetWrapper = vtkm::cont::SerializableDataSet; +using DataSetWrapper = vtkm::cont::DataSetWithCellSetTypes; -VTKM_CONT void TestEqualDataSet(const DataSetWrapper& ds1, const DataSetWrapper& ds2) +VTKM_CONT void TestEqualDataSetWrapper(const DataSetWrapper& ds1, const DataSetWrapper& ds2) { VTKM_TEST_ASSERT(test_equal_DataSets(ds1.DataSet, ds2.DataSet, CellSetTypes{})); } +VTKM_CONT void TestEqualDataSet(const vtkm::cont::DataSet& ds1, const vtkm::cont::DataSet& ds2) +{ + VTKM_TEST_ASSERT(test_equal_DataSets(ds1, ds2, CellSetTypes{})); +} + void RunTest(const vtkm::cont::DataSet& ds) { - TestSerialization(DataSetWrapper(ds), TestEqualDataSet); + VTKM_DEPRECATED_SUPPRESS_BEGIN + TestSerialization(vtkm::cont::SerializableDataSet(ds), + TestEqualDataSetWrapper); + VTKM_DEPRECATED_SUPPRESS_END + TestSerialization(DataSetWrapper(ds), TestEqualDataSetWrapper); + TestSerialization(ds, TestEqualDataSet); } void TestDataSetSerialization() diff --git a/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h b/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h index c511aafb0..a43db23a3 100644 --- a/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h +++ b/vtkm/filter/scalar_topology/testing/TestingContourTreeUniformDistributedFilter.h @@ -314,9 +314,6 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( { // Mutiple ranks -> Some assembly required. Collect data // on rank 0, all other ranks return empty data sets - using FieldTypeList = vtkm::ListAppend>; - using DataSetWrapper = - vtkm::cont::SerializableDataSet; // Communicate results to rank 0 auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator(); @@ -331,8 +328,7 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( p.enqueue(root, result.GetNumberOfPartitions()); for (const vtkm::cont::DataSet& curr_ds : result) { - auto curr_sds = DataSetWrapper(curr_ds); - p.enqueue(root, curr_sds); + p.enqueue(root, curr_ds); } }); // Exchange data, i.e., send to rank 0 (pass "true" to exchange data between @@ -352,9 +348,9 @@ inline vtkm::cont::PartitionedDataSet RunContourTreeDUniformDistributed( for (vtkm::Id currReceiveDataSetNo = 0; currReceiveDataSetNo < numberOfDataSetsToReceive; ++currReceiveDataSetNo) { - vtkm::cont::SerializableDataSet<> sds; - p.dequeue({ receiveFromRank, receiveFromRank }, sds); - combined_result.AppendPartition(sds.DataSet); + vtkm::cont::DataSet dsIncoming; + p.dequeue({ receiveFromRank, receiveFromRank }, dsIncoming); + combined_result.AppendPartition(dsIncoming); } } });