From 8f1c453c0b8f7d768484978cf30bfc4015b82357 Mon Sep 17 00:00:00 2001 From: Kenneth Moreland Date: Mon, 11 Oct 2021 12:41:30 -0600 Subject: [PATCH] Support writing binary files to legacy VTK files The legacy VTK file writer writes out in ASCII. This is helpful when a human is trying to read the file. However, if you have more than a trivial amount of data, the file can get impractically large. To get around this, `VTKDataSetWriter` now has a flag that allows you to write the data in binary format. --- docs/changelog/write-binary.md | 7 + vtkm/io/VTKDataSetWriter.cxx | 220 ++++++++++++++++--- vtkm/io/VTKDataSetWriter.h | 19 ++ vtkm/io/testing/UnitTestVTKDataSetWriter.cxx | 18 ++ 4 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 docs/changelog/write-binary.md diff --git a/docs/changelog/write-binary.md b/docs/changelog/write-binary.md new file mode 100644 index 000000000..866d07381 --- /dev/null +++ b/docs/changelog/write-binary.md @@ -0,0 +1,7 @@ +# Support writing binary files to legacy VTK files + +The legacy VTK file writer writes out in ASCII. This is helpful when a +human is trying to read the file. However, if you have more than a +trivial amount of data, the file can get impractically large. To get +around this, `VTKDataSetWriter` now has a flag that allows you to write +the data in binary format. diff --git a/vtkm/io/VTKDataSetWriter.cxx b/vtkm/io/VTKDataSetWriter.cxx index 2ea3b6be3..7527acff8 100644 --- a/vtkm/io/VTKDataSetWriter.cxx +++ b/vtkm/io/VTKDataSetWriter.cxx @@ -21,6 +21,7 @@ #include +#include #include #include @@ -85,10 +86,28 @@ using ArrayHandleRectilinearCoordinates = struct OutputArrayDataFunctor { template - VTKM_CONT void operator()(T, const vtkm::cont::UnknownArrayHandle& array, std::ostream& out) const + VTKM_CONT void operator()(T, + const vtkm::cont::UnknownArrayHandle& array, + std::ostream& out, + vtkm::io::FileType fileType) const { auto componentArray = array.ExtractArrayFromComponents(); auto portal = componentArray.ReadPortal(); + switch (fileType) + { + case vtkm::io::FileType::ASCII: + this->OutputAsciiArray(portal, out); + break; + case vtkm::io::FileType::BINARY: + this->OutputBinaryArray(portal, out); + break; + } + } + + template + void OutputAsciiArray(const PortalType& portal, std::ostream& out) const + { + using T = typename PortalType::ValueType::ComponentType; vtkm::Id numValues = portal.GetNumberOfValues(); for (vtkm::Id valueIndex = 0; valueIndex < numValues; ++valueIndex) @@ -109,11 +128,37 @@ struct OutputArrayDataFunctor out << "\n"; } } + + template + void OutputBinaryArray(const PortalType& portal, std::ostream& out) const + { + using T = typename PortalType::ValueType::ComponentType; + + vtkm::Id numValues = portal.GetNumberOfValues(); + std::vector tuple; + for (vtkm::Id valueIndex = 0; valueIndex < numValues; ++valueIndex) + { + auto value = portal.Get(valueIndex); + tuple.resize(static_cast(value.GetNumberOfComponents())); + for (vtkm::IdComponent cIndex = 0; cIndex < value.GetNumberOfComponents(); ++cIndex) + { + tuple[static_cast(cIndex)] = value[cIndex]; + } + if (vtkm::io::internal::IsLittleEndian()) + { + vtkm::io::internal::FlipEndianness(tuple); + } + out.write(reinterpret_cast(tuple.data()), + static_cast(tuple.size() * sizeof(T))); + } + } }; -void OutputArrayData(const vtkm::cont::UnknownArrayHandle& array, std::ostream& out) +void OutputArrayData(const vtkm::cont::UnknownArrayHandle& array, + std::ostream& out, + vtkm::io::FileType fileType) { - CallForBaseType(OutputArrayDataFunctor{}, array, out); + CallForBaseType(OutputArrayDataFunctor{}, array, out, fileType); } struct GetFieldTypeNameFunctor @@ -147,7 +192,7 @@ void WriteDimensions(std::ostream& out, const vtkm::cont::CellSetStructured out << (DIM > 2 ? VTraits::GetComponent(pointDimensions, 2) : 1) << "\n"; } -void WritePoints(std::ostream& out, const vtkm::cont::DataSet& dataSet) +void WritePoints(std::ostream& out, const vtkm::cont::DataSet& dataSet, vtkm::io::FileType fileType) { ///\todo: support other coordinate systems int cindex = 0; @@ -158,11 +203,11 @@ void WritePoints(std::ostream& out, const vtkm::cont::DataSet& dataSet) vtkm::Id npoints = cdata.GetNumberOfValues(); out << "POINTS " << npoints << " " << typeName << " " << '\n'; - OutputArrayData(cdata, out); + OutputArrayData(cdata, out, fileType); } template -void WriteExplicitCells(std::ostream& out, const CellSetType& cellSet) +void WriteExplicitCellsAscii(std::ostream& out, const CellSetType& cellSet) { vtkm::Id nCells = cellSet.GetNumberOfCells(); @@ -182,7 +227,9 @@ void WriteExplicitCells(std::ostream& out, const CellSetType& cellSet) out << nids; auto IdPortal = ids.ReadPortal(); for (int j = 0; j < nids; ++j) + { out << " " << IdPortal.Get(j); + } out << '\n'; } @@ -194,7 +241,74 @@ void WriteExplicitCells(std::ostream& out, const CellSetType& cellSet) } } -void WritePointFields(std::ostream& out, const vtkm::cont::DataSet& dataSet) +template +void WriteExplicitCellsBinary(std::ostream& out, const CellSetType& cellSet) +{ + vtkm::Id nCells = cellSet.GetNumberOfCells(); + + vtkm::Id conn_length = 0; + for (vtkm::Id i = 0; i < nCells; ++i) + { + conn_length += 1 + cellSet.GetNumberOfPointsInCell(i); + } + + out << "CELLS " << nCells << " " << conn_length << '\n'; + + std::vector buffer; + buffer.reserve(static_cast(conn_length)); + for (vtkm::Id i = 0; i < nCells; ++i) + { + vtkm::cont::ArrayHandle ids; + vtkm::Id nids = cellSet.GetNumberOfPointsInCell(i); + cellSet.GetIndices(i, ids); + buffer.push_back(static_cast(nids)); + auto IdPortal = ids.ReadPortal(); + for (int j = 0; j < nids; ++j) + { + buffer.push_back(static_cast(IdPortal.Get(j))); + } + } + if (vtkm::io::internal::IsLittleEndian()) + { + vtkm::io::internal::FlipEndianness(buffer); + } + VTKM_ASSERT(static_cast(buffer.size()) == conn_length); + out.write(reinterpret_cast(buffer.data()), + static_cast(buffer.size() * sizeof(vtkm::Int32))); + + out << "CELL_TYPES " << nCells << '\n'; + buffer.resize(0); + buffer.reserve(static_cast(nCells)); + for (vtkm::Id i = 0; i < nCells; ++i) + { + buffer.push_back(static_cast(cellSet.GetCellShape(i))); + } + if (vtkm::io::internal::IsLittleEndian()) + { + vtkm::io::internal::FlipEndianness(buffer); + } + VTKM_ASSERT(static_cast(buffer.size()) == nCells); + out.write(reinterpret_cast(buffer.data()), + static_cast(buffer.size() * sizeof(vtkm::Int32))); +} + +template +void WriteExplicitCells(std::ostream& out, const CellSetType& cellSet, vtkm::io::FileType fileType) +{ + switch (fileType) + { + case vtkm::io::FileType::ASCII: + WriteExplicitCellsAscii(out, cellSet); + break; + case vtkm::io::FileType::BINARY: + WriteExplicitCellsBinary(out, cellSet); + break; + } +} + +void WritePointFields(std::ostream& out, + const vtkm::cont::DataSet& dataSet, + vtkm::io::FileType fileType) { bool wrote_header = false; for (vtkm::Id f = 0; f < dataSet.GetNumberOfFields(); f++) @@ -227,11 +341,13 @@ void WritePointFields(std::ostream& out, const vtkm::cont::DataSet& dataSet) out << "SCALARS " << name << " " << typeName << " " << ncomps << '\n'; out << "LOOKUP_TABLE default" << '\n'; - OutputArrayData(field.GetData(), out); + OutputArrayData(field.GetData(), out, fileType); } } -void WriteCellFields(std::ostream& out, const vtkm::cont::DataSet& dataSet) +void WriteCellFields(std::ostream& out, + const vtkm::cont::DataSet& dataSet, + vtkm::io::FileType fileType) { bool wrote_header = false; for (vtkm::Id f = 0; f < dataSet.GetNumberOfFields(); f++) @@ -266,18 +382,19 @@ void WriteCellFields(std::ostream& out, const vtkm::cont::DataSet& dataSet) out << "SCALARS " << name << " " << typeName << " " << ncomps << '\n'; out << "LOOKUP_TABLE default" << '\n'; - OutputArrayData(field.GetData(), out); + OutputArrayData(field.GetData(), out, fileType); } } template void WriteDataSetAsUnstructured(std::ostream& out, const vtkm::cont::DataSet& dataSet, - const CellSetType& cellSet) + const CellSetType& cellSet, + vtkm::io::FileType fileType) { out << "DATASET UNSTRUCTURED_GRID" << '\n'; - WritePoints(out, dataSet); - WriteExplicitCells(out, cellSet); + WritePoints(out, dataSet, fileType); + WriteExplicitCells(out, cellSet, fileType); } template @@ -299,7 +416,8 @@ void WriteDataSetAsStructuredPoints(std::ostream& out, template void WriteDataSetAsRectilinearGrid(std::ostream& out, const ArrayHandleRectilinearCoordinates& points, - const vtkm::cont::CellSetStructured& cellSet) + const vtkm::cont::CellSetStructured& cellSet, + vtkm::io::FileType fileType) { out << "DATASET RECTILINEAR_GRID\n"; @@ -310,33 +428,35 @@ void WriteDataSetAsRectilinearGrid(std::ostream& out, dimArray = points.GetFirstArray(); out << "X_COORDINATES " << dimArray.GetNumberOfValues() << " " << typeName << "\n"; - OutputArrayData(dimArray, out); + OutputArrayData(dimArray, out, fileType); dimArray = points.GetSecondArray(); out << "Y_COORDINATES " << dimArray.GetNumberOfValues() << " " << typeName << "\n"; - OutputArrayData(dimArray, out); + OutputArrayData(dimArray, out, fileType); dimArray = points.GetThirdArray(); out << "Z_COORDINATES " << dimArray.GetNumberOfValues() << " " << typeName << "\n"; - OutputArrayData(dimArray, out); + OutputArrayData(dimArray, out, fileType); } template void WriteDataSetAsStructuredGrid(std::ostream& out, const vtkm::cont::DataSet& dataSet, - const vtkm::cont::CellSetStructured& cellSet) + const vtkm::cont::CellSetStructured& cellSet, + vtkm::io::FileType fileType) { out << "DATASET STRUCTURED_GRID" << '\n'; WriteDimensions(out, cellSet); - WritePoints(out, dataSet); + WritePoints(out, dataSet, fileType); } template void WriteDataSetAsStructured(std::ostream& out, const vtkm::cont::DataSet& dataSet, - const vtkm::cont::CellSetStructured& cellSet) + const vtkm::cont::CellSetStructured& cellSet, + vtkm::io::FileType fileType) { ///\todo: support rectilinear @@ -351,21 +471,27 @@ void WriteDataSetAsStructured(std::ostream& out, else if (coordSystem.IsType>()) { WriteDataSetAsRectilinearGrid( - out, coordSystem.AsArrayHandle>(), cellSet); + out, + coordSystem.AsArrayHandle>(), + cellSet, + fileType); } else if (coordSystem.IsType>()) { WriteDataSetAsRectilinearGrid( - out, coordSystem.AsArrayHandle>(), cellSet); + out, + coordSystem.AsArrayHandle>(), + cellSet, + fileType); } else { // Curvilinear is written as "structured grid" - WriteDataSetAsStructuredGrid(out, dataSet, cellSet); + WriteDataSetAsStructuredGrid(out, dataSet, cellSet, fileType); } } -void Write(std::ostream& out, const vtkm::cont::DataSet& dataSet) +void Write(std::ostream& out, const vtkm::cont::DataSet& dataSet, vtkm::io::FileType fileType) { // The Paraview parser cannot handle scientific notation: out << std::fixed; @@ -380,41 +506,54 @@ void Write(std::ostream& out, const vtkm::cont::DataSet& dataSet) #endif out << "# vtk DataFile Version 3.0" << '\n'; out << "vtk output" << '\n'; - out << "ASCII" << '\n'; + switch (fileType) + { + case vtkm::io::FileType::ASCII: + out << "ASCII" << '\n'; + break; + case vtkm::io::FileType::BINARY: + out << "BINARY" << '\n'; + break; + } vtkm::cont::DynamicCellSet cellSet = dataSet.GetCellSet(); if (cellSet.IsType>()) { - WriteDataSetAsUnstructured(out, dataSet, cellSet.Cast>()); + WriteDataSetAsUnstructured( + out, dataSet, cellSet.Cast>(), fileType); } else if (cellSet.IsType>()) { - WriteDataSetAsStructured(out, dataSet, cellSet.Cast>()); + WriteDataSetAsStructured( + out, dataSet, cellSet.Cast>(), fileType); } else if (cellSet.IsType>()) { - WriteDataSetAsStructured(out, dataSet, cellSet.Cast>()); + WriteDataSetAsStructured( + out, dataSet, cellSet.Cast>(), fileType); } else if (cellSet.IsType>()) { - WriteDataSetAsStructured(out, dataSet, cellSet.Cast>()); + WriteDataSetAsStructured( + out, dataSet, cellSet.Cast>(), fileType); } else if (cellSet.IsType>()) { // these function just like explicit cell sets - WriteDataSetAsUnstructured(out, dataSet, cellSet.Cast>()); + WriteDataSetAsUnstructured( + out, dataSet, cellSet.Cast>(), fileType); } else if (cellSet.IsType()) { - WriteDataSetAsUnstructured(out, dataSet, cellSet.Cast()); + WriteDataSetAsUnstructured(out, dataSet, cellSet.Cast(), fileType); } else { throw vtkm::cont::ErrorBadType("Could not determine type to write out."); } - WritePointFields(out, dataSet); - WriteCellFields(out, dataSet); + WritePointFields(out, dataSet, fileType); + WriteCellFields(out, dataSet, fileType); } } // anonymous namespace @@ -443,8 +582,8 @@ void VTKDataSetWriter::WriteDataSet(const vtkm::cont::DataSet& dataSet) const } try { - std::ofstream fileStream(this->FileName.c_str(), std::fstream::trunc); - Write(fileStream, dataSet); + std::ofstream fileStream(this->FileName.c_str(), std::fstream::trunc | std::fstream::binary); + Write(fileStream, dataSet, this->GetFileType()); fileStream.close(); } catch (std::ofstream::failure& error) @@ -452,5 +591,16 @@ void VTKDataSetWriter::WriteDataSet(const vtkm::cont::DataSet& dataSet) const throw vtkm::io::ErrorIO(error.what()); } } + +vtkm::io::FileType VTKDataSetWriter::GetFileType() const +{ + return this->FileType; +} + +void VTKDataSetWriter::SetFileType(vtkm::io::FileType type) +{ + this->FileType = type; +} + } } // namespace vtkm::io diff --git a/vtkm/io/VTKDataSetWriter.h b/vtkm/io/VTKDataSetWriter.h index f4dee8c76..4b50b0d49 100644 --- a/vtkm/io/VTKDataSetWriter.h +++ b/vtkm/io/VTKDataSetWriter.h @@ -19,6 +19,13 @@ namespace vtkm namespace io { +// Might want to place this somewhere else. +enum struct FileType +{ + ASCII, + BINARY +}; + struct VTKM_IO_EXPORT VTKDataSetWriter { public: @@ -27,8 +34,20 @@ public: VTKM_CONT void WriteDataSet(const vtkm::cont::DataSet& dataSet) const; + /// \brief Get whether the file will be written in ASCII or binary format. + /// + VTKM_CONT vtkm::io::FileType GetFileType() const; + + /// \{ + /// \brief Set whether the file will be written in ASCII or binary format. + VTKM_CONT void SetFileType(vtkm::io::FileType type); + VTKM_CONT void SetFileTypeToAscii() { this->SetFileType(vtkm::io::FileType::ASCII); } + VTKM_CONT void SetFileTypeToBinary() { this->SetFileType(vtkm::io::FileType::BINARY); } + /// \} + private: std::string FileName; + vtkm::io::FileType FileType = vtkm::io::FileType::ASCII; }; //struct VTKDataSetWriter } diff --git a/vtkm/io/testing/UnitTestVTKDataSetWriter.cxx b/vtkm/io/testing/UnitTestVTKDataSetWriter.cxx index 9de907d03..3df30b9dd 100644 --- a/vtkm/io/testing/UnitTestVTKDataSetWriter.cxx +++ b/vtkm/io/testing/UnitTestVTKDataSetWriter.cxx @@ -143,6 +143,24 @@ void TestVTKWriteTestData(const std::string& methodName, const vtkm::cont::DataS // Read back and check. vtkm::io::VTKDataSetReader reader(methodName + ".vtk"); CheckWrittenReadData(data, reader.ReadDataSet()); + + std::cout << "Writing " << methodName << " ascii" << std::endl; + vtkm::io::VTKDataSetWriter writerAscii(methodName + "-ascii.vtk"); + writerAscii.SetFileTypeToAscii(); + writerAscii.WriteDataSet(data); + + // Read back and check. + vtkm::io::VTKDataSetReader readerAscii(methodName + "-ascii.vtk"); + CheckWrittenReadData(data, readerAscii.ReadDataSet()); + + std::cout << "Writing " << methodName << " binary" << std::endl; + vtkm::io::VTKDataSetWriter writerBinary(methodName + "-binary.vtk"); + writerBinary.SetFileTypeToBinary(); + writerBinary.WriteDataSet(data); + + // Read back and check. + vtkm::io::VTKDataSetReader readerBinary(methodName + "-binary.vtk"); + CheckWrittenReadData(data, readerBinary.ReadDataSet()); } void TestVTKExplicitWrite()