From ba3571f740526eec053d6ddc9c416c6ce4005f45 Mon Sep 17 00:00:00 2001 From: Dave Pugmire Date: Wed, 28 Jun 2023 07:58:22 -0400 Subject: [PATCH 1/3] More efficient GetActiveParticles() --- vtkm/filter/flow/internal/AdvectAlgorithm.h | 99 ++++++++++++------- .../flow/internal/AdvectAlgorithmThreaded.h | 5 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/vtkm/filter/flow/internal/AdvectAlgorithm.h b/vtkm/filter/flow/internal/AdvectAlgorithm.h index eb6afab71..ae5114131 100644 --- a/vtkm/filter/flow/internal/AdvectAlgorithm.h +++ b/vtkm/filter/flow/internal/AdvectAlgorithm.h @@ -40,11 +40,11 @@ public: { } - void Execute(vtkm::Id numSteps, + void Execute(vtkm::Id maxNumSteps, vtkm::FloatDefault stepSize, const vtkm::cont::ArrayHandle& seeds) { - this->SetNumberOfSteps(numSteps); + this->SetMaxNumberOfSteps(maxNumSteps); this->SetStepSize(stepSize); this->SetSeeds(seeds); this->Go(); @@ -65,7 +65,7 @@ public: } void SetStepSize(vtkm::FloatDefault stepSize) { this->StepSize = stepSize; } - void SetNumberOfSteps(vtkm::Id numSteps) { this->NumberOfSteps = numSteps; } + void SetMaxNumberOfSteps(vtkm::Id numSteps) { this->MaxNumberOfSteps = numSteps; } void SetSeeds(const vtkm::cont::ArrayHandle& seeds) { this->ClearParticles(); @@ -100,8 +100,7 @@ public: vtkm::filter::flow::internal::ParticleMessenger messenger( this->Comm, this->UseAsynchronousCommunication, this->BoundsMap, 1, 128); - vtkm::Id nLocal = static_cast(this->Active.size() + this->Inactive.size()); - this->ComputeTotalNumParticles(nLocal); + this->ComputeTotalNumParticles(); while (this->TotalNumTerminatedParticles < this->TotalNumParticles) { @@ -113,7 +112,7 @@ public: auto& block = this->GetDataSet(blockId); DSIHelperInfoType bb = DSIHelperInfo(v, this->BoundsMap, this->ParticleBlockIDsMap); - block.Advect(bb, this->StepSize, this->NumberOfSteps); + block.Advect(bb, this->StepSize, this->MaxNumberOfSteps); numTerm = this->UpdateResult(bb.Get>()); } @@ -133,14 +132,17 @@ public: this->ParticleBlockIDsMap.clear(); } - void ComputeTotalNumParticles(const vtkm::Id& numLocal) + void ComputeTotalNumParticles() { - long long total = static_cast(numLocal); + vtkm::Id numLocal = static_cast(this->Inactive.size()); + for (const auto& it : this->Active) + numLocal += it.second.size(); + #ifdef VTKM_ENABLE_MPI - MPI_Comm mpiComm = vtkmdiy::mpi::mpi_cast(this->Comm.handle()); - MPI_Allreduce(MPI_IN_PLACE, &total, 1, MPI_LONG_LONG, MPI_SUM, mpiComm); + vtkmdiy::mpi::all_reduce(this->Comm, numLocal, this->TotalNumParticles, std::plus{}); +#else + this->TotalNumParticles = numLocal; #endif - this->TotalNumParticles = static_cast(total); } DataSetIntegrator& GetDataSet(vtkm::Id id) @@ -161,35 +163,56 @@ public: auto bit = blockIds.begin(); while (pit != particles.end() && bit != blockIds.end()) { + vtkm::Id blockId0 = (*bit)[0]; this->ParticleBlockIDsMap[pit->GetID()] = *bit; + if (this->Active.find(blockId0) == this->Active.end()) + this->Active[blockId0] = { *pit }; + else + this->Active[blockId0].emplace_back(*pit); pit++; bit++; } - - //Update Active - this->Active.insert(this->Active.end(), particles.begin(), particles.end()); } virtual bool GetActiveParticles(std::vector& particles, vtkm::Id& blockId) { - //DRP: particles = std::move(this->Active2[blockId]); particles.clear(); blockId = -1; if (this->Active.empty()) return false; - blockId = this->ParticleBlockIDsMap[this->Active.front().GetID()][0]; - auto it = this->Active.begin(); - while (it != this->Active.end()) + //If only one, return it. + if (this->Active.size() == 1) { - auto p = *it; - if (blockId == this->ParticleBlockIDsMap[p.GetID()][0]) + blockId = this->Active.begin()->first; + particles = std::move(this->Active.begin()->second); + this->Active.clear(); + } + else + { + //Find the blockId with the most particles. + std::size_t maxNum = 0; + auto maxIt = this->Active.end(); + for (auto it = this->Active.begin(); it != this->Active.end(); it++) { - particles.emplace_back(p); - it = this->Active.erase(it); + auto sz = it->second.size(); + if (sz > maxNum) + { + maxNum = sz; + maxIt = it; + } } - else - it++; + + if (maxNum == 0) + { + this->Active.clear(); + return false; + } + + VTKM_ASSERT(maxIt != this->Active.end()); + blockId = maxIt->first; + particles = std::move(maxIt->second); + this->Active.erase(maxIt); } return !particles.empty(); @@ -291,22 +314,27 @@ public: virtual void UpdateActive(const std::vector& particles, const std::unordered_map>& idsMap) { - this->Update(this->Active, particles, idsMap); + VTKM_ASSERT(particles.size() == idsMap.size()); + + for (auto pit = particles.begin(); pit != particles.end(); pit++) + { + vtkm::Id particleID = pit->GetID(); + const auto& it = idsMap.find(particleID); + VTKM_ASSERT(it != idsMap.end() && !it->second.empty()); + vtkm::Id blockId = it->second[0]; + this->Active[blockId].emplace_back(*pit); + } + + for (const auto& it : idsMap) + this->ParticleBlockIDsMap[it.first] = it.second; } virtual void UpdateInactive(const std::vector& particles, const std::unordered_map>& idsMap) - { - this->Update(this->Inactive, particles, idsMap); - } - - void Update(std::vector& arr, - const std::vector& particles, - const std::unordered_map>& idsMap) { VTKM_ASSERT(particles.size() == idsMap.size()); - arr.insert(arr.end(), particles.begin(), particles.end()); + this->Inactive.insert(this->Inactive.end(), particles.begin(), particles.end()); for (const auto& it : idsMap) this->ParticleBlockIDsMap[it.first] = it.second; } @@ -353,12 +381,13 @@ public: } //Member data - std::vector Active; + // {blockId, std::vector of particles} + std::unordered_map> Active; std::vector Blocks; vtkm::filter::flow::internal::BoundsMap BoundsMap; vtkmdiy::mpi::communicator Comm = vtkm::cont::EnvironmentTracker::GetCommunicator(); std::vector Inactive; - vtkm::Id NumberOfSteps; + vtkm::Id MaxNumberOfSteps; vtkm::Id NumRanks; //{particleId : {block IDs}} std::unordered_map> ParticleBlockIDsMap; diff --git a/vtkm/filter/flow/internal/AdvectAlgorithmThreaded.h b/vtkm/filter/flow/internal/AdvectAlgorithmThreaded.h index 81597b0d0..23dd2b2f6 100644 --- a/vtkm/filter/flow/internal/AdvectAlgorithmThreaded.h +++ b/vtkm/filter/flow/internal/AdvectAlgorithmThreaded.h @@ -48,8 +48,7 @@ public: void Go() override { - vtkm::Id nLocal = static_cast(this->Active.size() + this->Inactive.size()); - this->ComputeTotalNumParticles(nLocal); + this->ComputeTotalNumParticles(); std::vector workerThreads; workerThreads.emplace_back(std::thread(AdvectAlgorithmThreaded::Worker, this)); @@ -124,7 +123,7 @@ protected: auto& block = this->GetDataSet(blockId); DSIHelperInfoType bb = DSIHelperInfo(v, this->BoundsMap, this->ParticleBlockIDsMap); - block.Advect(bb, this->StepSize, this->NumberOfSteps); + block.Advect(bb, this->StepSize, this->MaxNumberOfSteps); this->UpdateWorkerResult(blockId, bb); } else From 9acb482a2631ea45001f86ea6c1bdc95065f61a5 Mon Sep 17 00:00:00 2001 From: Dave Pugmire Date: Wed, 28 Jun 2023 08:00:31 -0400 Subject: [PATCH 2/3] init maxnumsteps --- vtkm/filter/flow/internal/AdvectAlgorithm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtkm/filter/flow/internal/AdvectAlgorithm.h b/vtkm/filter/flow/internal/AdvectAlgorithm.h index ae5114131..75fe2e743 100644 --- a/vtkm/filter/flow/internal/AdvectAlgorithm.h +++ b/vtkm/filter/flow/internal/AdvectAlgorithm.h @@ -387,7 +387,7 @@ public: vtkm::filter::flow::internal::BoundsMap BoundsMap; vtkmdiy::mpi::communicator Comm = vtkm::cont::EnvironmentTracker::GetCommunicator(); std::vector Inactive; - vtkm::Id MaxNumberOfSteps; + vtkm::Id MaxNumberOfSteps = 0; vtkm::Id NumRanks; //{particleId : {block IDs}} std::unordered_map> ParticleBlockIDsMap; From c109d3dde72558d5d09378b381408607ed59d62e Mon Sep 17 00:00:00 2001 From: Dave Pugmire Date: Thu, 29 Jun 2023 08:24:55 -0400 Subject: [PATCH 3/3] kick the dashboards. --- vtkm/filter/flow/internal/AdvectAlgorithm.h | 1 - 1 file changed, 1 deletion(-) diff --git a/vtkm/filter/flow/internal/AdvectAlgorithm.h b/vtkm/filter/flow/internal/AdvectAlgorithm.h index 75fe2e743..b0fee4a15 100644 --- a/vtkm/filter/flow/internal/AdvectAlgorithm.h +++ b/vtkm/filter/flow/internal/AdvectAlgorithm.h @@ -209,7 +209,6 @@ public: return false; } - VTKM_ASSERT(maxIt != this->Active.end()); blockId = maxIt->first; particles = std::move(maxIt->second); this->Active.erase(maxIt);