diff --git a/pmd-dcpd/etc/changelog.txt b/pmd-dcpd/etc/changelog.txt index ef544f596f..c78e45504a 100644 --- a/pmd-dcpd/etc/changelog.txt +++ b/pmd-dcpd/etc/changelog.txt @@ -1,3 +1,6 @@ +???? 2002 - 0.3: +Faster still - now the workers can being working before all the batches are written to the space. + September 11 2001 - 0.2: Much, much faster now. java.lang on my basic farm of 3 machines takes 21 minutes. diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Batch.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Batch.java index 9dfbeb964f..bb039a6644 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Batch.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Batch.java @@ -18,13 +18,13 @@ public class Batch implements Entry { // payload public List tileWrappers; - public Integer jobID; + public Job job; public Integer sequenceID; public Batch() {} - public Batch(Integer jobID, List tileWrappers, Integer isDone, Integer sequenceID) { - this.jobID = jobID; + public Batch(Job job, List tileWrappers, Integer isDone, Integer sequenceID) { + this.job = job; this.tileWrappers = tileWrappers; this.isDone = isDone; this.sequenceID = sequenceID; diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/BatchBuilder.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/BatchBuilder.java index 7f43ef19ba..b90fbab9b5 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/BatchBuilder.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/BatchBuilder.java @@ -35,7 +35,7 @@ public class BatchBuilder { currentBatchSize++; if (wrappers.size() > maxBatchSize) { - Batch batch = new Batch(job.id, wrappers, Batch.NOT_DONE, new Integer(batches.size())); + Batch batch = new Batch(job, wrappers, Batch.NOT_DONE, new Integer(batches.size())); batches.add(batch); currentBatchSize = 0; @@ -44,7 +44,7 @@ public class BatchBuilder { } if (currentBatchSize > 0) { - Batch batch = new Batch(job.id, wrappers, Batch.NOT_DONE, new Integer(batches.size())); + Batch batch = new Batch(job, wrappers, Batch.NOT_DONE, new Integer(batches.size())); batches.add(batch); } diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPD.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPD.java index a999e5682d..e551e5eded 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPD.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPD.java @@ -36,7 +36,7 @@ public class DCPD { System.out.println("Tokenizing"); job = new Job("java_lang", new Integer(1)); - tokenSetWrapper = new TokenSetsWrapper(loadTokens("C:\\j2sdk1.4.0_01\\src\\java\\lang", true), job.id); + tokenSetWrapper = new TokenSetsWrapper(loadTokens("C:\\j2sdk1.4.0_01\\src\\java\\lang\\ref", true), job); System.out.println("Tokenizing complete, " + (System.currentTimeMillis()-start) + " elapsed ms"); System.out.println("Writing the TokenSetsWrapper to the space"); diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPDWorker.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPDWorker.java index 2236ec1a0f..06b98fa0c8 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPDWorker.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPDWorker.java @@ -21,9 +21,8 @@ import java.util.*; public class DCPDWorker { - private Job currentJob; - private TokenSetsWrapper tsw; private JavaSpace space; + private Map jobs = new HashMap(); public DCPDWorker(JavaSpace space) { try { @@ -42,14 +41,17 @@ public class DCPDWorker { public void jobAdded(Job job) { try { - currentJob = job; - System.out.println("Received a job " + currentJob.name + ", id is " + currentJob.id.intValue()); + System.out.println("Received a job " + job.name + ", id is " + job.id.intValue()); + if (!jobs.containsKey(job)) { + TokenSetsWrapper tsw = (TokenSetsWrapper)space.read(new TokenSetsWrapper(null, job), null, 100); + System.out.println("Read a TokenSetsWrapper with " + tsw.tokenSets.size() + " token lists"); + jobs.put(job, tsw); + } + - tsw = (TokenSetsWrapper)space.read(new TokenSetsWrapper(null, currentJob.id), null, 100); - System.out.println("Read a TokenSetsWrapper with " + tsw.tokenSets.size() + " token lists"); while (true) { - TileExpander te = new TileExpander(space, tsw); + TileExpander te = new TileExpander(space, jobs); te.expandAvailableTiles(); Thread.currentThread().yield(); } diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DGST.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DGST.java index 48b8701aa2..fe4de85e7e 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DGST.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/DGST.java @@ -32,11 +32,11 @@ public class DGST { public Results crunch(CPDListener listener) { Occurrences occ = new Occurrences(tokenSets, listener); try { + space.write(job, null, Lease.FOREVER); BatchBuilder builder = new BatchBuilder(occ, job); List batches = builder.buildBatches(); TilePlanter planter = new TilePlanter(space, job); planter.plant(batches); - space.write(job, null, Lease.FOREVER); expand(occ, batches.size()); System.out.println("Done"); } catch (Exception e) { diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Job.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Job.java index 052967440f..7f6f955756 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Job.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/Job.java @@ -17,4 +17,17 @@ public class Job implements Entry { this.id = id; this.name = name; } + + public boolean equals(Object o) { + Job other = (Job)o; + return other.id.equals(id); + } + + public int hashCode() { + return id.hashCode(); + } + + public String toString() { + return id + ":" + name; + } } diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileExpander.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileExpander.java index 3ddb4c7dfa..bf2c03552e 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileExpander.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileExpander.java @@ -16,29 +16,31 @@ import java.rmi.RemoteException; import java.util.Iterator; import java.util.List; import java.util.ArrayList; +import java.util.Map; public class TileExpander { private JavaSpace space; - private TokenSetsWrapper tsw; + private Map jobs; - public TileExpander(JavaSpace space, TokenSetsWrapper tsw) { + public TileExpander(JavaSpace space, Map jobs) { this.space = space; - this.tsw = tsw; + this.jobs = jobs; } public void expandAvailableTiles() throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{ - Entry twQuery = space.snapshot(new Batch(tsw.jobID, null, Batch.NOT_DONE, null)); + Entry twQuery = space.snapshot(new Batch(null, null, Batch.NOT_DONE, null)); Batch batch = null; int total = 0; while ((batch = (Batch)space.take(twQuery, null, 250)) != null) { + TokenSetsWrapper tsw = (TokenSetsWrapper)jobs.get(batch.job); total++; List wrappers = new ArrayList(); for (int j=0; j0) System.out.println("Expanded " + total + " tiles"); } - private Occurrences expand(TileWrapper tileWrapper, int sequenceID) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{ + private Occurrences expand(TokenSetsWrapper tsw, TileWrapper tileWrapper, int sequenceID) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{ Occurrences newOcc = new Occurrences(new CPDNullListener()); for (Iterator i = tileWrapper.occurrences.iterator(); i.hasNext();) { TokenEntry tok = (TokenEntry)i.next(); diff --git a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileHarvester.java b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileHarvester.java index 516d9a8673..3a5cd517b0 100644 --- a/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileHarvester.java +++ b/pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileHarvester.java @@ -29,7 +29,7 @@ public class TileHarvester { public Occurrences harvest(int batchesToHarvest) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException { Occurrences occ = new Occurrences(new CPDNullListener()); for (int i=0;i