Shuenhoy/vtk-m
1
0
Fork 0
mirror of https://gitlab.kitware.com/vtk/vtk-m synced 2024-09-08 13:23:51 +00:00
Code Issues 2 Actions Packages Projects Releases Wiki Activity

Save block extents and block index to files and read them instead of approximating them

Browse Source
This commit is contained in:
Gunther H. Weber 2020-09-25 16:22:24 -07:00
parent bee398d44a
commit c247c7d0e3
4 changed files with 55 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
examples/contour_tree_distributed/ContourTreeApp.cxx
View File

@ -481,6 +481,22 @@ int main(int argc, char* argv[])
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(bpd[0], bpd[1]);
getline(inFile, line);
std::istringstream blockIndex_stream(line);
blockIndex_stream >> tag;
if (tag != "#BLOCK_INDEX")
{
std::cerr << "Error: Expected #BLOCK_INDEX, got " << tag << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
std::vector<vtkm::Id> blockIndex;
while (blockIndex_stream >> dimVertices)
blockIndex.push_back(dimVertices);
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(blockIndex[0], blockIndex[1]);
getline(inFile, line);
std::istringstream linestream(line);
std::vector<vtkm::Id> dims;
@ -567,16 +583,10 @@ int main(int argc, char* argv[])
// and add to partition
useDataSet.AppendPartition(ds);
// FIXME: Hack: Approximate block index (better to put this into file)
localBlockIndicesPortal.Set(
blockNo,
vtkm::Id3{ static_cast<vtkm::Id>(
std::ceil(static_cast<float>(offset[0]) / static_cast<float>(dims[0]))),
static_cast<vtkm::Id>(
std::ceil(static_cast<float>(offset[1]) / static_cast<float>(dims[1]))),
static_cast<vtkm::Id>(nDims == 3 ? std::ceil(static_cast<float>(offset[2]) /
static_cast<float>(dims[2]))
: 0) });
localBlockIndicesPortal.Set(blockNo,
vtkm::Id3{ static_cast<vtkm::Id>(blockIndex[0]),
static_cast<vtkm::Id>(blockIndex[1]),
static_cast<vtkm::Id>(nDims == 3 ? blockIndex[2] : 0) });
localBlockOriginsPortal.Set(blockNo,
vtkm::Id3{ static_cast<vtkm::Id>(offset[0]),
static_cast<vtkm::Id>(offset[1]),
@ -597,6 +607,15 @@ int main(int argc, char* argv[])
static_cast<float>(dims[2]))
: 0) }
<< std::endl;
std::cout << "blockOrigin: "
<< vtkm::Id3{ static_cast<vtkm::Id>(offset[0]),
static_cast<vtkm::Id>(offset[1]),
static_cast<vtkm::Id>(nDims == 3 ? offset[2] : 0) }
<< std::endl;
std::cout << "blockSize: "
<< vtkm::Id3{ static_cast<vtkm::Id>(dims[0]),
static_cast<vtkm::Id>(dims[1]),
static_cast<vtkm::Id>(nDims == 3 ? dims[2] : 0) };
#endif
if (blockNo == 0)
@ -871,11 +890,8 @@ int main(int argc, char* argv[])
vtkm::worklet::contourtree_distributed::TreeCompiler treeCompiler;
treeCompiler.AddHierarchicalTree(result.GetPartition(ds_no));
char fname[256];
std::snprintf(fname,
sizeof(fname),
"TreeComplilerOutput_Rank%d_Block%d.dat",
rank,
static_cast<int>(ds_no));
std::snprintf(
fname, sizeof(fname), "TreeCompilerOutput_Rank%d_Block%d.dat", rank, static_cast<int>(ds_no));
FILE* out_file = std::fopen(fname, "wb");
treeCompiler.WriteBinary(out_file);
std::fclose(out_file);

9
examples/contour_tree_distributed/split_data_2d.py
View File

@ -11,11 +11,12 @@ def read_file(fn):
return data
# Save a block from a 2D NumPy array to disk
def save_piece(fn, array, offset, n_blocks, size):
def save_piece(fn, array, offset, n_blocks, block_index, size):
with open(fn, 'w') as f:
f.write('#GLOBAL_EXTENTS ' + ' '.join(map(str, array.shape)) + '\n')
f.write('#OFFSET ' + ' '.join(map(str, offset))+'\n')
f.write('#BLOCKS_PER_DIM ' + ' '.join(map(str, n_blocks))+'\n')
f.write('#BLOCK_INDEX ' + ' '.join(map(str, block_index))+'\n')
f.write(' '.join(map(str, size)) + '\n')
np.savetxt(f, array[offset[0]:offset[0]+size[0],offset[1]:offset[1]+size[1]], fmt='%.16g')
@ -49,10 +50,10 @@ split_points_y = split_points(data.shape[1], n_blocks[1])
# Save blocks
block_no = 0
for x_start, x_stop in zip(split_points_x, split_points_x[1:]):
for y_start, y_stop in zip(split_points_y, split_points_y[1:]):
for block_index_x, (x_start, x_stop) in enumerate(zip(split_points_x, split_points_x[1:])):
for block_index_y, (y_start, y_stop) in enumerate(zip(split_points_y, split_points_y[1:])):
n_x = x_stop - x_start + 1
n_y = y_stop - y_start + 1
save_piece(out_filename_pattern % block_no, data, (x_start, y_start), n_blocks, (n_x, n_y))
save_piece(out_filename_pattern % block_no, data, (x_start, y_start), n_blocks, (block_index_x, block_index_y), (n_x, n_y))
# print("Wrote block %d, origin %d %d, size %d %d" % (block_no, x_start, y_start, n_x, n_y))
block_no += 1

11
examples/contour_tree_distributed/split_data_3d.py
View File

@ -36,12 +36,13 @@ def readBOV(filename):
# Python order is slice, row, col
# TXT file order is row, col, slice
# offset and size are in file order
def save_piece(fn, array, offset, n_blocks, size):
def save_piece(fn, array, offset, n_blocks, block_index, size):
with open(fn, 'w') as f:
perm = [1, 2, 0]
f.write('#GLOBAL_EXTENTS ' + ' '.join(map(str, [array.shape[i] for i in perm])) + '\n')
f.write('#OFFSET ' + ' '.join(map(str, offset))+'\n')
f.write('#BLOCKS_PER_DIM ' + ' '.join(map(str, n_blocks))+'\n')
f.write('#BLOCK_INDEX ' + ' '.join(map(str, block_index))+'\n')
f.write(' '.join(map(str, size)) + '\n')
if fn[-5:]=='.bdem':
array[offset[2]:offset[2]+size[2],offset[0]:offset[0]+size[0],offset[1]:offset[1]+size[1]].astype(np.double).tofile(f)
@ -90,11 +91,11 @@ slice_filename = name + '_slices.txt'
# Save blocks
block_no = 0
for s_start, s_stop in zip(split_points_s, split_points_s[1:]):
for r_start, r_stop in zip(split_points_r, split_points_r[1:]):
for c_start, c_stop in zip(split_points_c, split_points_c[1:]):
for block_index_s, (s_start, s_stop) in enumerate(zip(split_points_s, split_points_s[1:])):
for block_index_r, (r_start, r_stop) in enumerate(zip(split_points_r, split_points_r[1:])):
for block_index_c, (c_start, c_stop) in enumerate(zip(split_points_c, split_points_c[)1:]):
n_s = s_stop - s_start + 1
n_r = r_stop - r_start + 1
n_c = c_stop - c_start + 1
save_piece(out_filename_pattern % block_no, data, (r_start, c_start, s_start), (n_r, n_c, n_s))
save_piece(out_filename_pattern % block_no, data, (r_start, c_start, s_start), (block_index_r, block_index_c, block_index_s), (n_r, n_c, n_s))
block_no += 1

26
examples/contour_tree_distributed/testrun.sh
View File

@ -13,67 +13,67 @@ echo "Starting Timing Runs"
echo
echo "8x9 Test Set"
./hact_test.sh $DATA_DIR/8x9test.txt 2
#./hact_test.sh $DATA_DIR/8x9test.txt 4
./hact_test.sh $DATA_DIR/8x9test.txt 4
# ./hact_test.sh $DATA_DIR/8x9test.txt 8
echo
echo "Vancouver Test Set"
./hact_test.sh $DATA_DIR/vanc.txt 2
#./hact_test.sh $DATA_DIR/vanc.txt 4
./hact_test.sh $DATA_DIR/vanc.txt 4
# ./hact_test.sh $DATA_DIR/vanc.txt 8
# ./hact_test.sh $DATA_DIR/vanc.txt 16
echo
echo "Vancouver SWSW Test Set"
./hact_test.sh $DATA_DIR/vancouverSWSW.txt 2
./hact_test.sh $DATA_DIR/vancouverSWSW.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSWSW.txt 8
./hact_test.sh $DATA_DIR/vancouverSWSW.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSWSW.txt 16
echo
echo "Vancouver SWNW Test Set"
./hact_test.sh $DATA_DIR/vancouverSWNW.txt 2
./hact_test.sh $DATA_DIR/vancouverSWNW.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSWNW.txt 8
./hact_test.sh $DATA_DIR/vancouverSWNW.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSWNW.txt 16
echo
echo "Vancouver SWSE Test Set"
./hact_test.sh $DATA_DIR/vancouverSWSE.txt 2
./hact_test.sh $DATA_DIR/vancouverSWSE.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSWSE.txt 8
./hact_test.sh $DATA_DIR/vancouverSWSE.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSWSE.txt 16
echo
echo "Vancouver SWNE Test Set"
./hact_test.sh $DATA_DIR/vancouverSWNE.txt 2
./hact_test.sh $DATA_DIR/vancouverSWNE.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSWNE.txt 8
./hact_test.sh $DATA_DIR/vancouverSWNE.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSWNE.txt 16
echo
echo "Vancouver NE Test Set"
./hact_test.sh $DATA_DIR/vancouverNE.txt 2
./hact_test.sh $DATA_DIR/vancouverNE.txt 4
# ./hact_test.sh $DATA_DIR/vancouverNE.txt 8
./hact_test.sh $DATA_DIR/vancouverNE.txt 8
# ./hact_test.sh $DATA_DIR/vancouverNE.txt 16
echo
echo "Vancouver NW Test Set"
./hact_test.sh $DATA_DIR/vancouverNW.txt 2
./hact_test.sh $DATA_DIR/vancouverNW.txt 4
# ./hact_test.sh $DATA_DIR/vancouverNW.txt 8
./hact_test.sh $DATA_DIR/vancouverNW.txt 8
# ./hact_test.sh $DATA_DIR/vancouverNW.txt 16
echo
echo "Vancouver SE Test Set"
./hact_test.sh $DATA_DIR/vancouverSE.txt 2
./hact_test.sh $DATA_DIR/vancouverSE.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSE.txt 8
./hact_test.sh $DATA_DIR/vancouverSE.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSE.txt 16
echo
echo "Vancouver SW Test Set"
./hact_test.sh $DATA_DIR/vancouverSW.txt 2
./hact_test.sh $DATA_DIR/vancouverSW.txt 4
# ./hact_test.sh $DATA_DIR/vancouverSW.txt 8
./hact_test.sh $DATA_DIR/vancouverSW.txt 8
# ./hact_test.sh $DATA_DIR/vancouverSW.txt 16
echo
echo "Icefields Test Set"
./hact_test.sh $DATA_DIR/icefield.txt 2
./hact_test.sh $DATA_DIR/icefield.txt 4
# ./hact_test.sh $DATA_DIR/icefield.txt 8
./hact_test.sh $DATA_DIR/icefield.txt 8
# ./hact_test.sh $DATA_DIR/icefield.txt 16
# ./hact_test.sh $DATA_DIR/icefield.txt 32
# ./hact_test.sh $DATA_DIR/icefield.txt 64
@ -81,7 +81,7 @@ echo
echo "GTOPO30 Full Tiny Test Set"
./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 2
./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 4
# ./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 8
./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 8
# ./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 16
# ./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 32
# ./hact_test.sh $DATA_DIR/gtopo_full_tiny.txt 64
@ -89,7 +89,7 @@ echo
echo "GTOPO30 UK Tile Test Set"
./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 2
./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 4
# ./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 8
./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 8
# ./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 16
# ./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 32
# ./hact_test.sh $DATA_DIR/gtopo30w020n40.txt 64