Shuenhoy/vtk-m
1
0
Fork 0
mirror of https://gitlab.kitware.com/vtk/vtk-m synced 2024-09-08 13:23:51 +00:00
Code Issues 2 Actions Packages Projects Releases Wiki Activity
049d0cca8c
vtk-m/examples/contour_tree_distributed/ContourTreeApp.cxx
Gunther H. Weber cd3fe3d47d Eliminate need for SpatialDecomposition in contour tree filters. 
This commit address issue #713 and removes the need to pass information
about the spatial decomposition (block origins, block sizes, block index
and blocks per dimension) to the contour tree filter. Block origin
information is added to CellSetStructured (as GlobalPointSize) and the
distributed contour tree filter will get this information from
CellSetStructured instead of expecting it as parameters to the
constructor. Information about blocks per dimension and block indices
are computed from the information in CellSetStructure albeit requiring
global communication across all ranks. To avoid this communication cost,
the caller of the filter may explicitly specify this information via the
SetBlockIndices() method.
2022-08-02 20:01:41 -07:00

1041 lines
39 KiB
C++
Raw Blame History

//============================================================================
// Copyright (c) Kitware, Inc.
// All rights reserved.
// See LICENSE.txt for details.
// This software is distributed WITHOUT ANY WARRANTY; without even
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the above copyright notice for more information.
//
// Copyright 2014 National Technology & Engineering Solutions of Sandia, LLC (NTESS).
// Copyright 2014 UT-Battelle, LLC.
// Copyright 2014 Los Alamos National Security.
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Under the terms of Contract DE-AC52-06NA25396 with Los Alamos National
// Laboratory (LANL), the U.S. Government retains certain rights in
// this software.
//============================================================================
// Copyright (c) 2018, The Regents of the University of California, through
// Lawrence Berkeley National Laboratory (subject to receipt of any required approvals
// from the U.S. Dept. of Energy). All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// (1) Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// (2) Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// (3) Neither the name of the University of California, Lawrence Berkeley National
// Laboratory, U.S. Dept. of Energy nor the names of its contributors may be
// used to endorse or promote products derived from this software without
// specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
// OF THE POSSIBILITY OF SUCH DAMAGE.
//
//=============================================================================
//
// This code is an extension of the algorithm presented in the paper:
// Parallel Peak Pruning for Scalable SMP Contour Tree Computation.
// Hamish Carr, Gunther Weber, Christopher Sewell, and James Ahrens.
// Proceedings of the IEEE Symposium on Large Data Analysis and Visualization
// (LDAV), October 2016, Baltimore, Maryland.
//
// The PPP2 algorithm and software were jointly developed by
// Hamish Carr (University of Leeds), Gunther H. Weber (LBNL), and
// Oliver Ruebel (LBNL)
//==============================================================================
#include <vtkm/Types.h>
#include <vtkm/cont/ArrayHandle.h>
#include <vtkm/cont/DataSet.h>
#include <vtkm/cont/DataSetBuilderUniform.h>
#include <vtkm/cont/DataSetFieldAdd.h>
#include <vtkm/cont/DeviceAdapterTag.h>
#include <vtkm/cont/Initialize.h>
#include <vtkm/cont/RuntimeDeviceTracker.h>
#include <vtkm/cont/Timer.h>
#include <vtkm/io/BOVDataSetReader.h>
#include <vtkm/filter/scalar_topology/ContourTreeUniformDistributed.h>
#include <vtkm/filter/scalar_topology/DistributedBranchDecompositionFilter.h>
#include <vtkm/filter/scalar_topology/worklet/branch_decomposition/HierarchicalVolumetricBranchDecomposer.h>
#include <vtkm/filter/scalar_topology/worklet/contourtree_augmented/PrintVectors.h>
#include <vtkm/filter/scalar_topology/worklet/contourtree_augmented/ProcessContourTree.h>
#include <vtkm/filter/scalar_topology/worklet/contourtree_augmented/Types.h>
#include <vtkm/filter/scalar_topology/worklet/contourtree_distributed/HierarchicalContourTree.h>
#include <vtkm/filter/scalar_topology/worklet/contourtree_distributed/TreeCompiler.h>
// clang-format off
VTKM_THIRDPARTY_PRE_INCLUDE
#include <vtkm/thirdparty/diy/Configure.h>
#include <vtkm/thirdparty/diy/diy.h>
VTKM_THIRDPARTY_POST_INCLUDE
// clang-format on
#include <mpi.h>
#include <cstdio>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
using ValueType = vtkm::Float64;
#define SINGLE_FILE_STDOUT_STDERR
// Simple helper class for parsing the command line options
class ParseCL
{
public:
ParseCL() {}
void parse(int& argc, char** argv)
{
mCLOptions.resize(static_cast<std::size_t>(argc));
for (std::size_t i = 1; i < static_cast<std::size_t>(argc); ++i)
{
this->mCLOptions[i] = std::string(argv[i]);
}
}
vtkm::Id findOption(const std::string& option) const
{
auto it =
std::find_if(this->mCLOptions.begin(),
this->mCLOptions.end(),
[option](const std::string& val) -> bool { return val.find(option) == 0; });
if (it == this->mCLOptions.end())
{
return -1;
}
else
{
return static_cast<vtkm::Id>(it - this->mCLOptions.begin());
}
}
bool hasOption(const std::string& option) const { return this->findOption(option) >= 0; }
std::string getOption(const std::string& option) const
{
std::size_t index = static_cast<std::size_t>(this->findOption(option));
std::string val = this->mCLOptions[index];
auto valPos = val.find("=");
if (valPos)
{
return val.substr(valPos + 1);
}
return std::string("");
}
const std::vector<std::string>& getOptions() const { return this->mCLOptions; }
private:
std::vector<std::string> mCLOptions;
};
// Compute and render an isosurface for a uniform grid example
int main(int argc, char* argv[])
{
// Set the logging levels we want to use. These could be made command line options as well
// Log level to be used for outputting timing information.
vtkm::cont::LogLevel timingsLogLevel = vtkm::cont::LogLevel::Info;
// Log level to be used for outputting metadata about the trees.
vtkm::cont::LogLevel treeLogLevel = vtkm::cont::LogLevel::Info;
// Log level for outputs specific to the example
vtkm::cont::LogLevel exampleLogLevel = vtkm::cont::LogLevel::Info;
// Setup the MPI environment.
MPI_Init(&argc, &argv);
auto comm = MPI_COMM_WORLD;
// Tell VTK-m which communicator it should use.
vtkm::cont::EnvironmentTracker::SetCommunicator(vtkmdiy::mpi::communicator());
// get the rank and size
int rank, size;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
// initialize vtkm-m (e.g., logging via -v and device via the -d option)
vtkm::cont::InitializeOptions vtkm_initialize_options =
vtkm::cont::InitializeOptions::RequireDevice;
vtkm::cont::InitializeResult vtkm_config =
vtkm::cont::Initialize(argc, argv, vtkm_initialize_options);
auto device = vtkm_config.Device;
VTKM_LOG_IF_S(exampleLogLevel, rank == 0, "Running with MPI. #ranks=" << size);
// Setup timing
vtkm::Float64 prevTime = 0;
vtkm::Float64 currTime = 0;
vtkm::cont::Timer totalTime;
totalTime.Start();
////////////////////////////////////////////
// Parse the command line options
////////////////////////////////////////////
ParseCL parser;
parser.parse(argc, argv);
std::string filename = parser.getOptions().back();
bool augmentHierarchicalTree = false;
if (parser.hasOption("--augmentHierarchicalTree"))
{
augmentHierarchicalTree = true;
}
bool computeHierarchicalVolumetricBranchDecomposition = false;
if (parser.hasOption("--computeVolumeBranchDecomposition"))
{
computeHierarchicalVolumetricBranchDecomposition = true;
if (!augmentHierarchicalTree)
{
VTKM_LOG_S(vtkm::cont::LogLevel::Warn,
"Warning: --computeVolumeBranchDecomposition only "
"allowed augmentation. Enabling --augmentHierarchicalTree option.");
augmentHierarchicalTree = true;
}
}
bool useBoundaryExtremaOnly = true;
if (parser.hasOption("--useFullBoundary"))
{
useBoundaryExtremaOnly = false;
}
bool useMarchingCubes = false;
if (parser.hasOption("--mc"))
{
useMarchingCubes = true;
if (useBoundaryExtremaOnly)
{
VTKM_LOG_S(vtkm::cont::LogLevel::Warn,
"Warning: Marching cubes connectivity currently only works when "
"using full boundary. Enabling the --useFullBoundary option "
"to ensure that the app works.");
useBoundaryExtremaOnly = false;
}
}
bool preSplitFiles = false;
if (parser.hasOption("--preSplitFiles"))
{
preSplitFiles = true;
}
bool saveDotFiles = false;
if (parser.hasOption("--saveDot"))
{
saveDotFiles = true;
}
bool saveOutputData = false;
if (parser.hasOption("--saveOutputData"))
{
saveOutputData = true;
}
bool forwardSummary = false;
if (parser.hasOption("--forwardSummary"))
{
forwardSummary = true;
}
int numBlocks = size;
int blocksPerRank = 1;
if (parser.hasOption("--numBlocks"))
{
numBlocks = std::stoi(parser.getOption("--numBlocks"));
if (numBlocks % size == 0)
blocksPerRank = numBlocks / size;
else
{
std::cerr << "Error: Number of blocks must be divisible by number of ranks." << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
}
if (argc < 2 || parser.hasOption("--help") || parser.hasOption("-h"))
{
if (rank == 0)
{
std::cout << "ContourTreeDistributed <options> <fileName>" << std::endl;
std::cout << std::endl;
std::cout << "<fileName> Name of the input data file." << std::endl;
std::cout << "The file is expected to be ASCII with either: " << std::endl;
std::cout << " - xdim ydim integers for 2D or" << std::endl;
std::cout << " - xdim ydim zdim integers for 3D" << std::endl;
std::cout << "followed by vector data last dimension varying fastest" << std::endl;
std::cout << std::endl;
std::cout << "----------------------------- VTKM Options -----------------------------"
<< std::endl;
std::cout << vtkm_config.Usage << std::endl;
std::cout << std::endl;
std::cout << "------------------------- Contour Tree Options -------------------------"
<< std::endl;
std::cout << "Options: (Bool options are give via int, i.e. =0 for False and =1 for True)"
<< std::endl;
std::cout << "--mc Use marching cubes connectivity (Default=False)." << std::endl;
std::cout << "--useFullBoundary Use the full boundary during. Typically only useful"
<< std::endl
<< " to compare the performance between using the full boundary"
<< std::endl;
std::cout << " and when using only boundary extrema." << std::endl;
std::cout << "--augmentHierarchicalTree Augment the hierarchical tree." << std::endl;
std::cout << "--computeVolumeBranchDecomposition Compute the volume branch decomposition. "
<< std::endl;
std::cout << " Requries --augmentHierarchicalTree to be set." << std::endl;
std::cout << "--preSplitFiles Input data is already pre-split into blocks." << std::endl;
std::cout << "--saveDot Save DOT files of the distributed contour tree " << std::endl
<< " computation (Default=False). " << std::endl;
std::cout << "--saveOutputData Save data files with hierarchical tree or volume data"
<< std::endl;
std::cout << "--numBlocks Number of blocks to use during computation "
<< "(Default=number of MPI ranks.)" << std::endl;
std::cout << "--forwardSummary Forward the summary timings also to the per-rank " << std::endl
<< " log files. Default is to round-robin print the " << std::endl
<< " summary instead" << std::endl;
std::cout << std::endl;
}
MPI_Finalize();
return EXIT_SUCCESS;
}
if (rank == 0)
{
VTKM_LOG_S(exampleLogLevel,
std::endl
<< " ------------ Settings -----------" << std::endl
<< " filename=" << filename << std::endl
<< " preSplitFiles=" << preSplitFiles << std::endl
<< " device=" << device.GetName() << std::endl
<< " mc=" << useMarchingCubes << std::endl
<< " useFullBoundary=" << !useBoundaryExtremaOnly << std::endl
<< " saveDot=" << saveDotFiles << std::endl
<< " augmentHierarchicalTree=" << augmentHierarchicalTree << std::endl
<< " computeVolumetricBranchDecomposition="
<< computeHierarchicalVolumetricBranchDecomposition << std::endl
<< " saveOutputData=" << saveOutputData << std::endl
<< " forwardSummary=" << forwardSummary << std::endl
<< " nblocks=" << numBlocks << std::endl);
}
// Redirect stdout to file if we are using MPI with Debugging
// From https://www.unix.com/302983597-post2.html
char cstr_filename[255];
std::snprintf(cstr_filename, sizeof(cstr_filename), "cout_%d.log", rank);
int out = open(cstr_filename, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (-1 == out)
{
perror("opening cout.log");
return 255;
}
#ifndef SINGLE_FILE_STDOUT_STDERR
std::snprintf(cstr_filename, sizeof(cstr_filename), "cerr_%d.log", rank);
int err = open(cstr_filename, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (-1 == err)
{
perror("opening cerr.log");
return 255;
}
#endif
int save_out = dup(fileno(stdout));
int save_err = dup(fileno(stderr));
if (-1 == dup2(out, fileno(stdout)))
{
perror("cannot redirect stdout");
return 255;
}
#ifdef SINGLE_FILE_STDOUT_STDERR
if (-1 == dup2(out, fileno(stderr)))
#else
if (-1 == dup2(err, fileno(stderr)))
#endif
{
perror("cannot redirect stderr");
return 255;
}
// Measure our time for startup
currTime = totalTime.GetElapsedTime();
vtkm::Float64 startUpTime = currTime - prevTime;
prevTime = currTime;
// Make sure that all ranks have started up before we start the data read
MPI_Barrier(comm);
currTime = totalTime.GetElapsedTime();
vtkm::Float64 startUpSyncTime = currTime - prevTime;
prevTime = currTime;
///////////////////////////////////////////////
// Read the input data
///////////////////////////////////////////////
vtkm::Float64 dataReadTime = 0;
vtkm::Float64 buildDatasetTime = 0;
std::vector<vtkm::Float32>::size_type nDims = 0;
// Multi-block dataset for multi-block DIY-paralle processing
vtkm::cont::PartitionedDataSet useDataSet;
// Domain decomposition information for DIY/contour tree
vtkm::Id3 globalSize;
vtkm::Id3 blocksPerDim;
vtkm::cont::ArrayHandle<vtkm::Id3> localBlockIndices;
localBlockIndices.Allocate(blocksPerRank);
auto localBlockIndicesPortal = localBlockIndices.WritePortal();
// Read the pre-split data files
if (preSplitFiles)
{
for (int blockNo = 0; blockNo < blocksPerRank; ++blockNo)
{
// Translate pattern into filename for this block
char block_filename[256];
snprintf(block_filename,
sizeof(block_filename),
filename.c_str(),
static_cast<int>(rank * blocksPerRank + blockNo));
std::cout << "Reading file " << block_filename << std::endl;
// Open file
std::ifstream inFile(block_filename);
if (!inFile.is_open() || inFile.bad())
{
std::cerr << "Error: Couldn't open file " << block_filename << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
// Read header with dimensions
std::string line;
std::string tag;
vtkm::Id dimVertices;
getline(inFile, line);
std::istringstream global_extents_stream(line);
global_extents_stream >> tag;
if (tag != "#GLOBAL_EXTENTS")
{
std::cerr << "Error: Expected #GLOBAL_EXTENTS, got " << tag << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
std::vector<vtkm::Id> global_extents;
while (global_extents_stream >> dimVertices)
global_extents.push_back(dimVertices);
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(global_extents[0], global_extents[1]);
if (blockNo == 0)
{ // First block: Set globalSize
globalSize =
vtkm::Id3{ static_cast<vtkm::Id>(global_extents[0]),
static_cast<vtkm::Id>(global_extents[1]),
static_cast<vtkm::Id>(global_extents.size() > 2 ? global_extents[2] : 1) };
}
else
{ // All other blocks: Consistency check of globalSize
if (globalSize !=
vtkm::Id3{ static_cast<vtkm::Id>(global_extents[0]),
static_cast<vtkm::Id>(global_extents[1]),
static_cast<vtkm::Id>(global_extents.size() > 2 ? global_extents[2] : 1) })
{
std::cerr << "Error: Global extents mismatch between blocks!" << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
}
getline(inFile, line);
std::istringstream offset_stream(line);
offset_stream >> tag;
if (tag != "#OFFSET")
{
std::cerr << "Error: Expected #OFFSET, got " << tag << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
std::vector<vtkm::Id> offset;
while (offset_stream >> dimVertices)
offset.push_back(dimVertices);
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(offset[0], offset[1]);
getline(inFile, line);
std::istringstream bpd_stream(line);
bpd_stream >> tag;
if (tag != "#BLOCKS_PER_DIM")
{
std::cerr << "Error: Expected #BLOCKS_PER_DIM, got " << tag << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
std::vector<vtkm::Id> bpd;
while (bpd_stream >> dimVertices)
bpd.push_back(dimVertices);
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(bpd[0], bpd[1]);
getline(inFile, line);
std::istringstream blockIndex_stream(line);
blockIndex_stream >> tag;
if (tag != "#BLOCK_INDEX")
{
std::cerr << "Error: Expected #BLOCK_INDEX, got " << tag << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
std::vector<vtkm::Id> blockIndex;
while (blockIndex_stream >> dimVertices)
blockIndex.push_back(dimVertices);
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(blockIndex[0], blockIndex[1]);
getline(inFile, line);
std::istringstream linestream(line);
std::vector<vtkm::Id> dims;
while (linestream >> dimVertices)
{
dims.push_back(dimVertices);
}
if (dims.size() != global_extents.size() || dims.size() != offset.size())
{
std::cerr << "Error: Dimension mismatch" << std::endl;
MPI_Finalize();
return EXIT_FAILURE;
}
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(dims[0], dims[1]);
// Compute the number of vertices, i.e., xdim * ydim * zdim
nDims = static_cast<unsigned short>(dims.size());
std::size_t numVertices = static_cast<std::size_t>(
std::accumulate(dims.begin(), dims.end(), std::size_t(1), std::multiplies<std::size_t>()));
// Check for fatal input errors
// Check that the number of dimensiosn is either 2D or 3D
bool invalidNumDimensions = (nDims < 2 || nDims > 3);
// Log any errors if found on rank 0
VTKM_LOG_IF_S(vtkm::cont::LogLevel::Error,
invalidNumDimensions && (rank == 0),
"The input mesh is " << nDims
<< "D. "
"The input data must be either 2D or 3D.");
// If we found any errors in the setttings than finalize MPI and exit the execution
if (invalidNumDimensions)
{
MPI_Finalize();
return EXIT_FAILURE;
}
// Read data
std::vector<ValueType> values(numVertices);
if (filename.compare(filename.length() - 5, 5, ".bdem") == 0)
{
inFile.read(reinterpret_cast<char*>(values.data()),
static_cast<std::streamsize>(numVertices * sizeof(ValueType)));
}
else
{
for (std::size_t vertex = 0; vertex < numVertices; ++vertex)
{
inFile >> values[vertex];
}
}
currTime = totalTime.GetElapsedTime();
dataReadTime = currTime - prevTime;
prevTime = currTime;
// Create vtk-m data set
vtkm::cont::DataSetBuilderUniform dsb;
vtkm::cont::DataSet ds;
if (nDims == 2)
{
const vtkm::Id2 v_dims{
static_cast<vtkm::Id>(dims[0]),
static_cast<vtkm::Id>(dims[1]),
};
const vtkm::Vec<ValueType, 2> v_origin{ static_cast<ValueType>(offset[0]),
static_cast<ValueType>(offset[1]) };
const vtkm::Vec<ValueType, 2> v_spacing{ 1, 1 };
ds = dsb.Create(v_dims, v_origin, v_spacing);
vtkm::cont::CellSetStructured<2> cs;
cs.SetPointDimensions(v_dims);
cs.SetGlobalPointDimensions(vtkm::Id2{ globalSize[0], globalSize[1] });
cs.SetGlobalPointIndexStart(vtkm::Id2{ offset[0], offset[1] });
ds.SetCellSet(cs);
}
else
{
VTKM_ASSERT(nDims == 3);
const vtkm::Id3 v_dims{ static_cast<vtkm::Id>(dims[0]),
static_cast<vtkm::Id>(dims[1]),
static_cast<vtkm::Id>(dims[2]) };
const vtkm::Vec<ValueType, 3> v_origin{ static_cast<ValueType>(offset[0]),
static_cast<ValueType>(offset[1]),
static_cast<ValueType>(offset[2]) };
vtkm::Vec<ValueType, 3> v_spacing(1, 1, 1);
ds = dsb.Create(v_dims, v_origin, v_spacing);
vtkm::cont::CellSetStructured<3> cs;
cs.SetPointDimensions(v_dims);
cs.SetGlobalPointDimensions(globalSize);
cs.SetGlobalPointIndexStart(vtkm::Id3{ offset[0], offset[1], offset[2] });
ds.SetCellSet(cs);
}
ds.AddPointField("values", values);
// and add to partition
useDataSet.AppendPartition(ds);
localBlockIndicesPortal.Set(
blockNo,
vtkm::Id3{ static_cast<vtkm::Id>(blockIndex[0]),
static_cast<vtkm::Id>(blockIndex[1]),
static_cast<vtkm::Id>(nDims == 3 ? blockIndex[2] : 0) });
if (blockNo == 0)
{
blocksPerDim = vtkm::Id3{ static_cast<vtkm::Id>(bpd[0]),
static_cast<vtkm::Id>(bpd[1]),
static_cast<vtkm::Id>(nDims == 3 ? bpd[2] : 1) };
}
}
// Print the mesh metadata
if (rank == 0)
{
VTKM_LOG_S(exampleLogLevel,
std::endl
<< " ---------------- Input Mesh Properties --------------" << std::endl
<< " Number of dimensions: " << nDims << std::endl);
}
}
// Read single-block data and split it for the ranks
else
{
vtkm::cont::DataSet inDataSet;
// Currently FloatDefualt would be fine, but it could cause problems if we ever
// read binary files here.
std::vector<ValueType> values;
std::vector<vtkm::Id> dims;
// Read BOV data file
if (filename.compare(filename.length() - 3, 3, "bov") == 0)
{
std::cout << "Reading BOV file" << std::endl;
vtkm::io::BOVDataSetReader reader(filename);
inDataSet = reader.ReadDataSet();
nDims = 3;
currTime = totalTime.GetElapsedTime();
dataReadTime = currTime - prevTime;
prevTime = currTime;
// Copy the data into the values array so we can construct a multiblock dataset
// TODO All we should need to do to implement BOV support is to copy the values
// in the values vector and copy the dimensions in the dims vector
vtkm::Id3 pointDimensions;
auto cellSet = inDataSet.GetCellSet();
vtkm::cont::CastAndCall(
cellSet, vtkm::worklet::contourtree_augmented::GetPointDimensions(), pointDimensions);
std::cout << "Point dimensions are " << pointDimensions << std::endl;
dims.resize(3);
dims[0] = pointDimensions[0];
dims[1] = pointDimensions[1];
dims[2] = pointDimensions[2];
auto tempFieldData = inDataSet.GetField(0).GetData();
values.resize(static_cast<std::size_t>(tempFieldData.GetNumberOfValues()));
auto valuesHandle = vtkm::cont::make_ArrayHandle(values, vtkm::CopyFlag::Off);
vtkm::cont::ArrayCopy(tempFieldData, valuesHandle);
valuesHandle.SyncControlArray(); //Forces values to get updated if copy happened on GPU
}
// Read ASCII data input
else
{
std::cout << "Reading ASCII file" << std::endl;
std::ifstream inFile(filename);
if (inFile.bad())
return 0;
// Read the dimensions of the mesh, i.e,. number of elementes in x, y, and z
std::string line;
getline(inFile, line);
std::istringstream linestream(line);
vtkm::Id dimVertices;
while (linestream >> dimVertices)
{
dims.push_back(dimVertices);
}
// Swap dimensions so that they are from fastest to slowest growing
// dims[0] -> col; dims[1] -> row, dims[2] ->slice
std::swap(dims[0], dims[1]);
// Compute the number of vertices, i.e., xdim * ydim * zdim
nDims = static_cast<unsigned short>(dims.size());
std::size_t numVertices = static_cast<std::size_t>(
std::accumulate(dims.begin(), dims.end(), std::size_t(1), std::multiplies<std::size_t>()));
// Check the the number of dimensiosn is either 2D or 3D
bool invalidNumDimensions = (nDims < 2 || nDims > 3);
// Log any errors if found on rank 0
VTKM_LOG_IF_S(vtkm::cont::LogLevel::Error,
invalidNumDimensions && (rank == 0),
"The input mesh is " << nDims << "D. The input data must be either 2D or 3D.");
// If we found any errors in the setttings than finalize MPI and exit the execution
if (invalidNumDimensions)
{
MPI_Finalize();
return EXIT_FAILURE;
}
// Read data
values.resize(numVertices);
for (std::size_t vertex = 0; vertex < numVertices; ++vertex)
{
inFile >> values[vertex];
}
// finish reading the data
inFile.close();
currTime = totalTime.GetElapsedTime();
dataReadTime = currTime - prevTime;
prevTime = currTime;
} // END ASCII Read
// Print the mesh metadata
if (rank == 0)
{
VTKM_LOG_S(exampleLogLevel,
std::endl
<< " ---------------- Input Mesh Properties --------------" << std::endl
<< " Number of dimensions: " << nDims);
}
// Create a multi-block dataset for multi-block DIY-paralle processing
blocksPerDim = nDims == 3 ? vtkm::Id3(1, 1, numBlocks)
: vtkm::Id3(1, numBlocks, 1); // Decompose the data into
globalSize = nDims == 3 ? vtkm::Id3(static_cast<vtkm::Id>(dims[0]),
static_cast<vtkm::Id>(dims[1]),
static_cast<vtkm::Id>(dims[2]))
: vtkm::Id3(static_cast<vtkm::Id>(dims[0]),
static_cast<vtkm::Id>(dims[1]),
static_cast<vtkm::Id>(1));
{
vtkm::Id lastDimSize =
(nDims == 2) ? static_cast<vtkm::Id>(dims[1]) : static_cast<vtkm::Id>(dims[2]);
if (size > (lastDimSize / 2.))
{
VTKM_LOG_IF_S(vtkm::cont::LogLevel::Error,
rank == 0,
"Number of ranks too large for data. Use " << lastDimSize / 2
<< "or fewer ranks");
MPI_Finalize();
return EXIT_FAILURE;
}
vtkm::Id standardBlockSize = (vtkm::Id)(lastDimSize / numBlocks);
vtkm::Id blockSize = standardBlockSize;
vtkm::Id blockSliceSize =
nDims == 2 ? static_cast<vtkm::Id>(dims[0]) : static_cast<vtkm::Id>((dims[0] * dims[1]));
vtkm::Id blockNumValues = blockSize * blockSliceSize;
vtkm::Id startBlock = blocksPerRank * rank;
vtkm::Id endBlock = startBlock + blocksPerRank;
for (vtkm::Id blockIndex = startBlock; blockIndex < endBlock; ++blockIndex)
{
vtkm::Id localBlockIndex = blockIndex - startBlock;
vtkm::Id blockStart = blockIndex * blockNumValues;
vtkm::Id blockEnd = blockStart + blockNumValues;
if (blockIndex < (numBlocks - 1)) // add overlap between regions
{
blockEnd += blockSliceSize;
}
else
{
blockEnd = lastDimSize * blockSliceSize;
}
vtkm::Id currBlockSize = (vtkm::Id)((blockEnd - blockStart) / blockSliceSize);
vtkm::cont::DataSetBuilderUniform dsb;
vtkm::cont::DataSet ds;
// 2D data
if (nDims == 2)
{
vtkm::Id2 vdims;
vdims[0] = static_cast<vtkm::Id>(dims[0]);
vdims[1] = static_cast<vtkm::Id>(currBlockSize);
vtkm::Vec<ValueType, 2> origin(0, blockIndex * blockSize);
vtkm::Vec<ValueType, 2> spacing(1, 1);
ds = dsb.Create(vdims, origin, spacing);
vtkm::cont::CellSetStructured<2> cs;
cs.SetPointDimensions(vdims);
cs.SetGlobalPointDimensions(vtkm::Id2{ globalSize[0], globalSize[1] });
cs.SetGlobalPointIndexStart(vtkm::Id2{ 0, (blockStart / blockSliceSize) });
ds.SetCellSet(cs);
localBlockIndicesPortal.Set(localBlockIndex, vtkm::Id3(0, blockIndex, 0));
}
// 3D data
else
{
vtkm::Id3 vdims;
vdims[0] = static_cast<vtkm::Id>(dims[0]);
vdims[1] = static_cast<vtkm::Id>(dims[1]);
vdims[2] = static_cast<vtkm::Id>(currBlockSize);
vtkm::Vec<ValueType, 3> origin(0, 0, (blockIndex * blockSize));
vtkm::Vec<ValueType, 3> spacing(1, 1, 1);
ds = dsb.Create(vdims, origin, spacing);
vtkm::cont::CellSetStructured<3> cs;
cs.SetPointDimensions(vdims);
cs.SetGlobalPointDimensions(globalSize);
cs.SetGlobalPointIndexStart(vtkm::Id3(0, 0, blockStart / blockSliceSize));
ds.SetCellSet(cs);
localBlockIndicesPortal.Set(localBlockIndex, vtkm::Id3(0, 0, blockIndex));
}
std::vector<vtkm::Float32> subValues((values.begin() + blockStart),
(values.begin() + blockEnd));
ds.AddPointField("values", subValues);
useDataSet.AppendPartition(ds);
}
}
}
// Check if marching cubes is enabled for non 3D data
bool invalidMCOption = (useMarchingCubes && nDims != 3);
VTKM_LOG_IF_S(vtkm::cont::LogLevel::Error,
invalidMCOption && (rank == 0),
"The input mesh is "
<< nDims << "D. "
<< "Contour tree using marching cubes is only supported for 3D data.");
// If we found any errors in the setttings than finalize MPI and exit the execution
if (invalidMCOption)
{
MPI_Finalize();
return EXIT_FAILURE;
}
currTime = totalTime.GetElapsedTime();
buildDatasetTime = currTime - prevTime;
prevTime = currTime;
// Make sure that all ranks have started up before we start the data read
MPI_Barrier(comm);
currTime = totalTime.GetElapsedTime();
vtkm::Float64 dataReadSyncTime = currTime - prevTime;
prevTime = currTime;
// Convert the mesh of values into contour tree, pairs of vertex ids
vtkm::filter::scalar_topology::ContourTreeUniformDistributed filter(timingsLogLevel,
treeLogLevel);
filter.SetBlockIndices(blocksPerDim, localBlockIndices);
filter.SetUseBoundaryExtremaOnly(useBoundaryExtremaOnly);
filter.SetUseMarchingCubes(useMarchingCubes);
filter.SetAugmentHierarchicalTree(augmentHierarchicalTree);
filter.SetSaveDotFiles(saveDotFiles);
filter.SetActiveField("values");
// Execute the contour tree analysis
auto result = filter.Execute(useDataSet);
currTime = totalTime.GetElapsedTime();
vtkm::Float64 computeContourTreeTime = currTime - prevTime;
prevTime = currTime;
// Make sure that all ranks have started up before we start the data read
MPI_Barrier(comm);
currTime = totalTime.GetElapsedTime();
vtkm::Float64 postFilterSyncTime = currTime - prevTime;
prevTime = currTime;
if (saveOutputData)
{
if (augmentHierarchicalTree)
{
if (computeHierarchicalVolumetricBranchDecomposition)
{
vtkm::filter::scalar_topology::DistributedBranchDecompositionFilter bd_filter;
auto bd_result = bd_filter.Execute(result);
for (vtkm::Id ds_no = 0; ds_no < result.GetNumberOfPartitions(); ++ds_no)
{
auto ds = bd_result.GetPartition(ds_no);
std::string branchDecompositionFileName = std::string("BranchDecomposition_Rank_") +
std::to_string(static_cast<int>(rank)) + std::string("_Block_") +
std::to_string(static_cast<int>(ds_no)) + std::string(".txt");
std::ofstream treeStream(branchDecompositionFileName.c_str());
treeStream
<< vtkm::filter::scalar_topology::HierarchicalVolumetricBranchDecomposer::PrintBranches(
ds);
}
}
else
{
for (vtkm::Id ds_no = 0; ds_no < result.GetNumberOfPartitions(); ++ds_no)
{
auto ds = result.GetPartition(ds_no);
vtkm::worklet::contourtree_augmented::IdArrayType supernodes;
ds.GetField("Supernodes").GetData().AsArrayHandle(supernodes);
vtkm::worklet::contourtree_augmented::IdArrayType superarcs;
ds.GetField("Superarcs").GetData().AsArrayHandle(superarcs);
vtkm::worklet::contourtree_augmented::IdArrayType regularNodeGlobalIds;
ds.GetField("RegularNodeGlobalIds").GetData().AsArrayHandle(regularNodeGlobalIds);
vtkm::Id totalVolume = globalSize[0] * globalSize[1] * globalSize[2];
vtkm::worklet::contourtree_augmented::IdArrayType intrinsicVolume;
ds.GetField("IntrinsicVolume").GetData().AsArrayHandle(intrinsicVolume);
vtkm::worklet::contourtree_augmented::IdArrayType dependentVolume;
ds.GetField("DependentVolume").GetData().AsArrayHandle(dependentVolume);
std::string dumpVolumesString =
vtkm::worklet::contourtree_distributed::HierarchicalContourTree<ValueType>::DumpVolumes(
supernodes,
superarcs,
regularNodeGlobalIds,
totalVolume,
intrinsicVolume,
dependentVolume);
std::string volumesFileName = std::string("TreeWithVolumes_Rank_") +
std::to_string(static_cast<int>(rank)) + std::string("_Block_") +
std::to_string(static_cast<int>(ds_no)) + std::string(".txt");
std::ofstream treeStream(volumesFileName.c_str());
treeStream << dumpVolumesString;
}
}
}
else
{
for (vtkm::Id ds_no = 0; ds_no < result.GetNumberOfPartitions(); ++ds_no)
{
vtkm::worklet::contourtree_distributed::TreeCompiler treeCompiler;
treeCompiler.AddHierarchicalTree(result.GetPartition(ds_no));
char fname[256];
std::snprintf(fname,
sizeof(fname),
"TreeCompilerOutput_Rank%d_Block%d.dat",
rank,
static_cast<int>(ds_no));
FILE* out_file = std::fopen(fname, "wb");
treeCompiler.WriteBinary(out_file);
std::fclose(out_file);
}
}
}
currTime = totalTime.GetElapsedTime();
vtkm::Float64 saveOutputDataTime = currTime - prevTime;
prevTime = currTime;
std::cout << std::flush;
std::cerr << std::flush;
close(out);
#ifndef SINGLE_FILE_STDOUT_STDERR
close(err);
#endif
if (!forwardSummary)
{
// write our log data now and close the files so that the summary is written round-robin to
// the normal log output stream
dup2(save_out, fileno(stdout));
dup2(save_err, fileno(stderr));
// close our MPI forward
close(save_out);
close(save_err);
// Force a simple round-robin on the ranks for the summary prints. Its not perfect for MPI but
// it works well enough to sort the summaries from the ranks for small-scale debugging.
if (rank > 0)
{
int temp;
MPI_Status status;
MPI_Recv(&temp, 1, MPI_INT, (rank - 1), 0, comm, &status);
}
}
currTime = totalTime.GetElapsedTime();
VTKM_LOG_S(timingsLogLevel,
std::endl
<< " -------------------------- Totals " << rank
<< " -----------------------------" << std::endl
<< std::setw(42) << std::left << " Start-up"
<< ": " << startUpTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Start-up Sync"
<< ": " << startUpSyncTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Data Read"
<< ": " << dataReadTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Build VTKM Dataset"
<< ": " << buildDatasetTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Data Read/Build Sync"
<< ": " << dataReadSyncTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Compute Contour Tree"
<< ": " << computeContourTreeTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Post filter Sync"
<< ": " << postFilterSyncTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Save Tree Compiler Data"
<< ": " << saveOutputDataTime << " seconds" << std::endl
<< std::setw(42) << std::left << " Total Time"
<< ": " << currTime << " seconds");
// Flush ouput streams just to make sure everything has been logged (in particular when using MPI)
std::cout << std::flush;
std::cerr << std::flush;
// Round robin output
if (!forwardSummary)
{
// Let the next rank know that it is time to print their summary.
if (rank < (size - 1))
{
int message = 1;
MPI_Send(&message, 1, MPI_INT, (rank + 1), 0, comm);
}
}
// Finish logging to our per-rank log-files
else
{
// write our log data now
// the normal log output stream
dup2(save_out, fileno(stdout));
dup2(save_err, fileno(stderr));
// close our MPI forward
close(save_out);
close(save_err);
}
// Finalize and finish the execution
MPI_Finalize();
return EXIT_SUCCESS;
}
Reference in New Issue View Git Blame Copy Permalink