From 9da2b86d7261a1e7c2ebb991283063dde8e8d6d6 Mon Sep 17 00:00:00 2001
From: Dave Pugmire <dpugmire@gmail.com>
Date: Sat, 21 Nov 2020 22:13:16 -0500
Subject: [PATCH] Implement blockAndWait for threaded

---
 .../particleadvection/AdvectorBaseAlgorithm.h |  3 +--
 .../AdvectorBaseThreadedAlgorithm.h           | 23 +++++++------------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/vtkm/filter/particleadvection/AdvectorBaseAlgorithm.h b/vtkm/filter/particleadvection/AdvectorBaseAlgorithm.h
index 288f71cbb..62a1c5998 100644
--- a/vtkm/filter/particleadvection/AdvectorBaseAlgorithm.h
+++ b/vtkm/filter/particleadvection/AdvectorBaseAlgorithm.h
@@ -341,10 +341,9 @@ protected:
     //2. numLocalTerm + this->TotalNumberOfTerminatedParticles == this->TotalNumberOfParticles
     //So, if neither are true, we can safely block and wait for communication to come in.
 
-    if (this->Active.empty() &&
+    if (this->Active.empty() && this->Inactive.empty() &&
         (numLocalTerm + this->TotalNumTerminatedParticles < this->TotalNumParticles))
     {
-      std::cout << " BLOCK!! rank= " << this->Rank << std::endl;
       return true;
     }
 
diff --git a/vtkm/filter/particleadvection/AdvectorBaseThreadedAlgorithm.h b/vtkm/filter/particleadvection/AdvectorBaseThreadedAlgorithm.h
index 5d395088f..a11e5bd68 100644
--- a/vtkm/filter/particleadvection/AdvectorBaseThreadedAlgorithm.h
+++ b/vtkm/filter/particleadvection/AdvectorBaseThreadedAlgorithm.h
@@ -56,7 +56,11 @@ protected:
   bool GetActiveParticles(std::vector<vtkm::Particle>& particles, vtkm::Id& blockId) override
   {
     std::lock_guard<std::mutex> lock(this->Mutex);
-    return this->AdvectorBaseAlgorithm<ResultType>::GetActiveParticles(particles, blockId);
+    bool val = this->AdvectorBaseAlgorithm<ResultType>::GetActiveParticles(particles, blockId);
+
+    this->WorkerIdle = !val;
+
+    return val;
   }
 
   void UpdateActive(const std::vector<vtkm::Particle>& particles,
@@ -77,8 +81,7 @@ protected:
 
   void WorkerWait()
   {
-    this->WorkerIdle = true;
-
+    //    this->WorkerIdle = true;
     //    std::cout<<"Worker wait..."<<std::endl;
     //    std::unique_lock<std::mutex> lock(this->WorkAvailMutex);
     //    this->WorkAvailableCondition.wait(lock);
@@ -92,7 +95,6 @@ protected:
       vtkm::Id blockId = -1;
       if (this->GetActiveParticles(v, blockId))
       {
-        this->WorkerIdle = false;
         const auto& block = this->GetDataSet(blockId);
 
         ResultType r;
@@ -139,25 +141,16 @@ protected:
     }
 
     //Let the workers know that we are done.
-    std::cout << this->Rank << " DONE" << std::endl;
     this->SetDone();
     //    this->WorkAvailableCondition.notify_all();
   }
 
   bool GetBlockAndWait(const vtkm::Id& numLocalTerm) override
   {
-    return false;
-    /*
     std::lock_guard<std::mutex> lock(this->Mutex);
-    bool val = this->AdvectorBaseAlgorithm<ResultType>::GetBlockAndWait(numLocalTerm);
-    if (val && this->WorkerIdle)
-        val = true;
-    else
-        val = false;
-    if (this->Rank == 0) std::cout<<" M: GBW: val= "<<val<<" ((wi= "<<this->WorkerIdle<<" Asz= "<<this->Active.size()<<std::endl;
 
-    return val;
-      */
+    return (this->AdvectorBaseAlgorithm<ResultType>::GetBlockAndWait(numLocalTerm) &&
+            this->WorkerIdle && this->WorkerResults.empty());
   }
 
   void GetWorkerResults(std::unordered_map<vtkm::Id, std::vector<ResultType>>& results)