Shuenhoy/vtk-m
1
0
Fork 0
mirror of https://gitlab.kitware.com/vtk/vtk-m synced 2024-10-05 01:49:02 +00:00
Code Issues 2 Actions Packages Projects Releases Wiki Activity

Merge topic 'fix-dataset-reader'

Browse Source 
57f516811 Support reading Global and Pedigree ids
a01e4335d Support reading new format cells
d7b2fec44 Fix reading of string arrays

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Kenneth Moreland <morelandkd@ornl.gov>
Merge-request: !2490
This commit is contained in:
Sujin Philip 2021-05-17 16:52:31 +00:00 committed by Kitware Robot
commit 0a401c2ace
8 changed files with 228 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
data/data/uniform/simple_structured_points_strings_ascii.vtk Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c20b10be38f4dd931bebbea201fb1d4b3b033c048f2e2c77be369bd58d9d0b0a
size 603

3
data/data/uniform/simple_structured_points_strings_bin.vtk Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:efa0d42dc02757126f2186f8c479e370f462c3cd7a16417240d967294725cdab
size 367

3
data/data/unstructured/simple_unstructured_ascii_v5.vtk Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a12c7f1989c34e670fdc3d1756f5f40a906fe065e6e0b356427f0601b5c90e77
size 1584

3
data/data/unstructured/simple_unstructured_bin_v5.vtk Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a9b2f98c364d25f6e5c10b85ffac733961e66d224e2a4842ad1f209af7c4501a
size 2418

185
vtkm/io/VTKDataSetReaderBase.cxx
View File

@ -11,7 +11,9 @@
#include <vtkm/io/VTKDataSetReaderBase.h>
#include <vtkm/VecTraits.h>
#include <vtkm/cont/ArrayCopy.h>
#include <vtkm/cont/ArrayHandle.h>
#include <vtkm/cont/ArrayHandleOffsetsToNumComponents.h>
#include <vtkm/cont/ArrayPortalToIterators.h>
#include <vtkm/cont/Logging.h>
#include <vtkm/cont/UnknownArrayHandle.h>
@ -258,25 +260,61 @@ void VTKDataSetReaderBase::ReadPoints()
void VTKDataSetReaderBase::ReadCells(vtkm::cont::ArrayHandle<vtkm::Id>& connectivity,
vtkm::cont::ArrayHandle<vtkm::IdComponent>& numIndices)
{
vtkm::Id numCells, numInts;
this->DataFile->Stream >> numCells >> numInts >> std::ws;
connectivity.Allocate(numInts - numCells);
numIndices.Allocate(numCells);
std::vector<vtkm::Int32> buffer(static_cast<std::size_t>(numInts));
this->ReadArray(buffer);
vtkm::Int32* buffp = buffer.data();
auto connectivityPortal = connectivity.WritePortal();
auto numIndicesPortal = numIndices.WritePortal();
for (vtkm::Id i = 0, connInd = 0; i < numCells; ++i)
if (this->DataFile->Version[0] < 5)
{
vtkm::IdComponent numInds = static_cast<vtkm::IdComponent>(*buffp++);
numIndicesPortal.Set(i, numInds);
for (vtkm::IdComponent j = 0; j < numInds; ++j, ++connInd)
vtkm::Id numCells, numInts;
this->DataFile->Stream >> numCells >> numInts >> std::ws;
connectivity.Allocate(numInts - numCells);
numIndices.Allocate(numCells);
std::vector<vtkm::Int32> buffer(static_cast<std::size_t>(numInts));
this->ReadArray(buffer);
vtkm::Int32* buffp = buffer.data();
auto connectivityPortal = connectivity.WritePortal();
auto numIndicesPortal = numIndices.WritePortal();
for (vtkm::Id i = 0, connInd = 0; i < numCells; ++i)
{
connectivityPortal.Set(connInd, static_cast<vtkm::Id>(*buffp++));
vtkm::IdComponent numInds = static_cast<vtkm::IdComponent>(*buffp++);
numIndicesPortal.Set(i, numInds);
for (vtkm::IdComponent j = 0; j < numInds; ++j, ++connInd)
{
connectivityPortal.Set(connInd, static_cast<vtkm::Id>(*buffp++));
}
}
}
else
{
vtkm::Id offsetsSize, connSize;
this->DataFile->Stream >> offsetsSize >> connSize >> std::ws;
std::string tag, dataType;
this->DataFile->Stream >> tag >> dataType >> std::ws;
internal::parseAssert(tag == "OFFSETS");
auto offsets =
this->DoReadArrayVariant(vtkm::cont::Field::Association::ANY, dataType, offsetsSize, 1);
offsets.CastAndCallForTypes<vtkm::List<vtkm::Int64, vtkm::Int32>,
vtkm::List<vtkm::cont::StorageTagBasic>>(
[&](const auto& offsetsAH) {
vtkm::cont::ArrayCopy(vtkm::cont::make_ArrayHandleOffsetsToNumComponents(
vtkm::cont::make_ArrayHandleCast(offsetsAH, vtkm::Id{})),
numIndices);
});
this->DataFile->Stream >> tag >> dataType >> std::ws;
internal::parseAssert(tag == "CONNECTIVITY");
auto conn =
this->DoReadArrayVariant(vtkm::cont::Field::Association::ANY, dataType, connSize, 1);
if (conn.IsValueType<vtkm::Id>())
{
conn.AsArrayHandle(connectivity);
}
else
{
conn.CastAndCallForTypes<vtkm::List<vtkm::Int64, vtkm::Int32>,
vtkm::List<vtkm::cont::StorageTagBasic>>(
[&](const auto& connAH) { vtkm::cont::ArrayCopy(connAH, connectivity); });
}
}
}
@ -322,6 +360,12 @@ void VTKDataSetReaderBase::ReadAttributes()
{
association = vtkm::cont::Field::Association::CELL_SET;
}
else if (tag == "FIELD") // can see field in this position also
{
this->ReadGlobalFields(nullptr);
this->DataFile->Stream >> tag;
continue;
}
else
{
internal::parseAssert(false);
@ -359,6 +403,10 @@ void VTKDataSetReaderBase::ReadAttributes()
{
this->ReadFields(association, size);
}
else if (tag == "GLOBAL_IDS" || tag == "PEDIGREE_IDS")
{
this->ReadGlobalOrPedigreeIds(association, size);
}
else
{
break;
@ -611,6 +659,22 @@ void VTKDataSetReaderBase::ReadGlobalFields(std::vector<vtkm::Float32>* visitBou
}
}
void VTKDataSetReaderBase::ReadGlobalOrPedigreeIds(vtkm::cont::Field::Association association,
std::size_t numElements)
{
std::string dataName;
std::string dataType;
this->DataFile->Stream >> dataName >> dataType >> std::ws;
internal::parseAssert(dataType == "vtkIdType");
std::vector<vtkm::Int32> buffer(numElements); // vtk writes vtkIdType as int
this->ReadArray(buffer);
vtkm::cont::UnknownArrayHandle data(vtkm::cont::make_ArrayHandleMove(std::move(buffer)));
this->AddField(dataName, association, data);
this->SkipArrayMetaData(1);
}
class VTKDataSetReaderBase::SkipArrayVariant
{
public:
@ -693,16 +757,11 @@ void VTKDataSetReaderBase::DoSkipArrayVariant(std::string dataType,
std::size_t numElements,
vtkm::IdComponent numComponents)
{
// string is unsupported for SkipArrayVariant, so it requires some
// special handling
if (dataType == "string")
// string requires some special handling
if (dataType == "string" || dataType == "utf8_string")
{
const vtkm::Id stringCount = numComponents * static_cast<vtkm::Id>(numElements);
for (vtkm::Id i = 0; i < stringCount; ++i)
{
std::string trash;
this->DataFile->Stream >> trash;
}
this->SkipStringArray(stringCount);
}
else
{
@ -722,9 +781,20 @@ vtkm::cont::UnknownArrayHandle VTKDataSetReaderBase::DoReadArrayVariant(
vtkm::cont::ArrayHandle<vtkm::Float32> empty;
vtkm::cont::UnknownArrayHandle data(empty);
vtkm::io::internal::DataType typeId = vtkm::io::internal::DataTypeId(dataType);
vtkm::io::internal::SelectTypeAndCall(
typeId, numComponents, ReadArrayVariant(this, association, numElements, data));
// string requires some special handling
if (dataType == "string" || dataType == "utf8_string")
{
VTKM_LOG_S(vtkm::cont::LogLevel::Warn,
"Support for data type 'string' and 'utf8_string' is not implemented. Skipping.");
const vtkm::Id stringCount = numComponents * static_cast<vtkm::Id>(numElements);
this->SkipStringArray(stringCount);
}
else
{
vtkm::io::internal::DataType typeId = vtkm::io::internal::DataTypeId(dataType);
vtkm::io::internal::SelectTypeAndCall(
typeId, numComponents, ReadArrayVariant(this, association, numElements, data));
}
return data;
}
@ -758,6 +828,65 @@ void VTKDataSetReaderBase::SkipArray(std::size_t numElements,
this->SkipArrayMetaData(numComponents);
}
void VTKDataSetReaderBase::SkipStringArray(std::size_t numStrings)
{
if (this->DataFile->IsBinary)
{
for (std::size_t i = 0; i < numStrings; ++i)
{
auto firstByte = this->DataFile->Stream.peek();
auto type = firstByte >> 6;
switch (type)
{
case 3: // length stored in 1 byte
{
auto length = this->DataFile->Stream.get();
length &= 0x3F;
this->DataFile->Stream.seekg(static_cast<std::streamoff>(length), std::ios_base::cur);
break;
}
case 2: // length stored in 2 bytes
{
vtkm::UInt16 length = 0;
auto bytes = reinterpret_cast<char*>(&length);
this->DataFile->Stream.read(bytes, 2);
std::swap(bytes[0], bytes[1]);
length &= 0x3FFF;
this->DataFile->Stream.seekg(static_cast<std::streamoff>(length), std::ios_base::cur);
break;
}
case 1: // length stored in 4 bytes
{
vtkm::UInt32 length = 0;
auto bytes = reinterpret_cast<char*>(&length);
this->DataFile->Stream.read(bytes, 4);
std::reverse(bytes, bytes + 4);
length &= 0x3FFFFFFF;
this->DataFile->Stream.seekg(static_cast<std::streamoff>(length), std::ios_base::cur);
break;
}
default: // length stored in 8 bytes
{
vtkm::UInt64 length = 0;
auto bytes = reinterpret_cast<char*>(&length);
this->DataFile->Stream.read(bytes, 8);
std::reverse(bytes, bytes + 8);
this->DataFile->Stream.seekg(static_cast<std::streamoff>(length), std::ios_base::cur);
break;
}
}
}
}
else
{
for (std::size_t i = 0; i < numStrings; ++i)
{
// ASCII mode stores one string per line
this->DataFile->Stream.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
}
}
}
void VTKDataSetReaderBase::SkipArrayMetaData(vtkm::IdComponent numComponents)
{
if (!this->DataFile->Stream.good())

4
vtkm/io/VTKDataSetReaderBase.h
View File

@ -168,6 +168,8 @@ private:
VTKM_CONT void ReadTensors(vtkm::cont::Field::Association association, std::size_t numElements);
VTKM_CONT void ReadFields(vtkm::cont::Field::Association association,
std::size_t expectedNumElements);
VTKM_CONT void ReadGlobalOrPedigreeIds(vtkm::cont::Field::Association association,
std::size_t numElements);
protected:
VTKM_CONT void ReadGlobalFields(std::vector<vtkm::Float32>* visitBounds = nullptr);
@ -271,6 +273,8 @@ protected:
vtkm::io::internal::DummyBitType,
vtkm::IdComponent numComponents = 1);
VTKM_CONT void SkipStringArray(std::size_t numStrings);
VTKM_CONT void SkipArrayMetaData(vtkm::IdComponent numComponents);
};
}

15
vtkm/io/internal/VTKDataSetTypes.h
View File

@ -37,14 +37,19 @@ enum DataType
DTYPE_UNSIGNED_LONG,
DTYPE_LONG,
DTYPE_FLOAT,
DTYPE_DOUBLE
DTYPE_DOUBLE,
DTYPE_UNSIGNED_LONG_LONG,
DTYPE_LONG_LONG,
DTYPE_COUNT
};
inline const char* DataTypeString(int id)
{
static const char* strings[] = {
"", "bit", "unsigned_char", "char", "unsigned_short", "short", "unsigned_int",
"int", "unsigned_long", "long", "float", "double"
"", "bit", "unsigned_char", "char", "unsigned_short",
"short", "unsigned_int", "int", "unsigned_long", "long",
"float", "double", "vtktypeuint64", "vtktypeint64"
};
return strings[id];
}
@ -52,7 +57,7 @@ inline const char* DataTypeString(int id)
inline DataType DataTypeId(const std::string& str)
{
DataType type = DTYPE_UNKNOWN;
for (int id = 1; id < 12; ++id)
for (int id = 1; id < DTYPE_COUNT; ++id)
{
if (str == DataTypeString(id))
{
@ -220,9 +225,11 @@ inline void SelectTypeAndCall(DataType dtype,
SelectVecTypeAndCall(vtkm::Int32(), numComponents, functor);
break;
case DTYPE_UNSIGNED_LONG:
case DTYPE_UNSIGNED_LONG_LONG:
SelectVecTypeAndCall(vtkm::UInt64(), numComponents, functor);
break;
case DTYPE_LONG:
case DTYPE_LONG_LONG:
SelectVecTypeAndCall(vtkm::Int64(), numComponents, functor);
break;
case DTYPE_FLOAT:

44
vtkm/io/testing/UnitTestVTKDataSetReader.cxx
View File

@ -125,6 +125,23 @@ void TestReadingUnstructuredGrid(Format format)
"Incorrect cellset type");
}
void TestReadingV5Format(Format format)
{
std::string testFileName = (format == FORMAT_ASCII)
? vtkm::cont::testing::Testing::DataPath("unstructured/simple_unstructured_ascii_v5.vtk")
: vtkm::cont::testing::Testing::DataPath("unstructured/simple_unstructured_bin_v5.vtk");
vtkm::cont::DataSet ds = readVTKDataSet(testFileName);
VTKM_TEST_ASSERT(ds.GetNumberOfFields() == 6, "Incorrect number of fields");
VTKM_TEST_ASSERT(ds.GetNumberOfPoints() == 26, "Incorrect number of points");
VTKM_TEST_ASSERT(ds.GetCellSet().GetNumberOfPoints() == 26,
"Incorrect number of points (from cell set)");
VTKM_TEST_ASSERT(ds.GetNumberOfCells() == 15, "Incorrect number of cells");
VTKM_TEST_ASSERT(ds.GetCellSet().IsType<vtkm::cont::CellSetExplicit<>>(),
"Incorrect cellset type");
}
void TestReadingUnstructuredGridEmpty()
{
vtkm::cont::DataSet data =
@ -468,6 +485,23 @@ void TestReadingFusion()
VTKM_TEST_ASSERT(test_equal(zVecRange.Max, 0.480726));
}
void TestSkppingStringFields(Format format)
{
std::string testFileName = (format == FORMAT_ASCII)
? vtkm::cont::testing::Testing::DataPath("uniform/simple_structured_points_strings_ascii.vtk")
: vtkm::cont::testing::Testing::DataPath("uniform/simple_structured_points_strings_bin.vtk");
vtkm::cont::DataSet ds = readVTKDataSet(testFileName);
VTKM_TEST_ASSERT(ds.GetNumberOfFields() == 1, "Incorrect number of fields");
VTKM_TEST_ASSERT(ds.GetNumberOfPoints() == 72, "Incorrect number of points");
VTKM_TEST_ASSERT(ds.GetCellSet().GetNumberOfPoints() == 72,
"Incorrect number of points (from cell set)");
VTKM_TEST_ASSERT(ds.GetNumberOfCells() == 30, "Incorrect number of cells");
VTKM_TEST_ASSERT(ds.GetCellSet().IsType<vtkm::cont::CellSetStructured<3>>(),
"Incorrect cellset type");
}
void TestReadingVTKDataSet()
{
std::cout << "Test reading VTK Polydata file in ASCII" << std::endl;
@ -509,6 +543,16 @@ void TestReadingVTKDataSet()
TestReadingASCIIFishTank();
std::cout << "Test reading fusion" << std::endl;
TestReadingFusion();
std::cout << "Test skipping string fields in ASCII files" << std::endl;
TestSkppingStringFields(FORMAT_ASCII);
std::cout << "Test skipping string fields in BINARY files" << std::endl;
TestSkppingStringFields(FORMAT_BINARY);
std::cout << "Test reading v5 file format in ASCII" << std::endl;
TestReadingV5Format(FORMAT_ASCII);
std::cout << "Test reading v5 file format in BINARY" << std::endl;
TestReadingV5Format(FORMAT_BINARY);
}
int UnitTestVTKDataSetReader(int argc, char* argv[])