hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

modified to distribute tiles in batches

Browse Source 
git-svn-id: https://pmd.svn.sourceforge.net/svnroot/pmd/trunk@907 51baf565-9d33-0410-a72c-fc3788e3496d
This commit is contained in:
Tom Copeland
2002-09-11 15:57:04 +00:00
parent c5b1975822
commit 2b4efa166c
12 changed files with 172 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pmd-dcpd/etc/build.xml
View File

@ -19,7 +19,7 @@
<classpath>
<pathelement location="${compiletimelibsdir}/jini-core.jar"/>
<pathelement location="${compiletimelibsdir}/jini-ext.jar"/>
<pathelement location="${lib}/pmd-0.9.jar"/>
<pathelement location="${lib}/pmd-1.0rc1.jar"/>
</classpath>
</javac>
</target>

2
pmd-dcpd/etc/go.bat
View File

@ -3,7 +3,7 @@ set CLASSPATH=
set CLASSPATH=%CLASSPATH%;c:\data\pmd\pmd-dcpd\lib\runtimeonly\outrigger-dl.jar
set CLASSPATH=%CLASSPATH%;c:\data\pmd\pmd-dcpd\lib\runtimeonly\reggie.jar
set CLASSPATH=%CLASSPATH%;c:\data\pmd\pmd-dcpd\build
set CLASSPATH=%CLASSPATH%;c:\data\pmd\pmd-dcpd\lib\pmd-0.9.jar
set CLASSPATH=%CLASSPATH%;c:\data\pmd\pmd-dcpd\lib\pmd-1.0rc1.jar
set MAIN=net.sourceforge.pmd.dcpd.%1
set MEMORY_ARG=-Xms128M -Xmx384M

4
pmd-dcpd/regress/test/net/sourceforge/pmd/dcpd/TileExpanderTest.java
View File

@ -24,6 +24,7 @@ public class TileExpanderTest extends TestCase {
}
public void test1() throws Throwable {
/*
TokenSets tokenSets = TileExpanderTest.createTokenSets();
MockJavaSpace space = new MockJavaSpace();
Job job = new Job("foo", new Integer(1));
@ -36,7 +37,7 @@ public class TileExpanderTest extends TestCase {
TileWrapper tw = new TileWrapper(tile,
marshal(occ.getOccurrences(tile)),
job.id,
TileWrapper.NOT_DONE,
Chunk.NOT_DONE,
null,
new Integer(tilesSoFar),
null, null);
@ -52,6 +53,7 @@ public class TileExpanderTest extends TestCase {
List writtenEntries = space.getWrittenEntries();
assertEquals(7, writtenEntries.size());
*/
}
public static List marshal(Iterator i) {

4
pmd-dcpd/regress/test/net/sourceforge/pmd/dcpd/TileHarvesterTest.java
View File

@ -22,6 +22,7 @@ public class TileHarvesterTest extends TestCase {
}
public void test1() throws Throwable {
/*
TokenSets tokenSets = TileExpanderTest.createTokenSets();
MockJavaSpace space = new MockJavaSpace();
Job job = new Job("foo", new Integer(1));
@ -36,7 +37,7 @@ public class TileHarvesterTest extends TestCase {
TileWrapper tw = new TileWrapper(tile,
TileExpanderTest.marshal(occ.getOccurrences(tile)),
job.id,
TileWrapper.NOT_DONE,
Chunk.NOT_DONE,
null,
new Integer(tilesSoFar),
null, null);
@ -52,5 +53,6 @@ public class TileHarvesterTest extends TestCase {
TileHarvester tileGatherer = new TileHarvester(space, job);
Occurrences newOcc = tileGatherer.harvest(occ.size());
assertEquals(2, newOcc.size());
*/
}
}

33
pmd-dcpd/src/net/sourceforge/pmd/dcpd/Chunk.java Normal file
View File

@ -0,0 +1,33 @@
/*
* User: tom
* Date: Sep 10, 2002
* Time: 2:35:32 PM
*/
package net.sourceforge.pmd.dcpd;
import net.jini.core.entry.Entry;
import java.util.List;
public class Chunk implements Entry {
// routing information
public static final Integer DONE = new Integer( 1 );
public static final Integer NOT_DONE = new Integer( 0 );
public Integer isDone;
// payload
public List tileWrappers;
public Integer jobID;
public Integer sequenceID;
public Chunk() {}
public Chunk(Integer jobID, List tileWrappers, Integer isDone, Integer sequenceID) {
this.jobID = jobID;
this.tileWrappers = tileWrappers;
this.isDone = isDone;
this.sequenceID = sequenceID;
}
}

2
pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPD.java
View File

@ -44,7 +44,7 @@ public class DCPD {
System.out.println("Writing complete, " + (System.currentTimeMillis()-start) + " elapsed ms");
System.out.println("Crunching");
DGST dgst = new DGST(space, job, tokenSetWrapper.tokenSets, 100);
DGST dgst = new DGST(space, job, tokenSetWrapper.tokenSets, 30);
Results results = dgst.crunch(new CPDListenerImpl());
System.out.println("Crunching complete, " + (System.currentTimeMillis()-start) + " elapsed ms");

79
pmd-dcpd/src/net/sourceforge/pmd/dcpd/DCPDResultsImpl.java Normal file
View File

@ -0,0 +1,79 @@
/*
* User: tom
* Date: Sep 11, 2002
* Time: 11:35:19 AM
*/
package net.sourceforge.pmd.dcpd;
import net.sourceforge.pmd.cpd.*;
import java.io.Serializable;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
public class DCPDResultsImpl extends TileOccurrences implements Results, Serializable {
public void addTile(Tile tile, TokenEntry tok) {
super.addTile(tile, tok);
for (int i=orderedTiles.size()-1; i>=0; i--) {
Tile candidate = (Tile)orderedTiles.get(i);
removeDupesOf(candidate);
}
}
public int getTileLineCount(Tile tile, TokenSets tokenSets) {
TokenEntry firstToken = (TokenEntry)((List)tileToOccurrenceMap.get(tile)).get(0);
TokenList tl = tokenSets.getTokenList(firstToken);
TokenEntry lastToken = (TokenEntry)tl.get(firstToken.getIndex()-1 + tile.getTokenCount());
return lastToken.getBeginLine() - firstToken.getBeginLine() - 1;
}
private void removeDupesOf(Tile largerTile) {
for (int i=0; i<orderedTiles.size()-1; i++) {
Tile smallerTile = (Tile)orderedTiles.get(i);
outer:
for (int j=0; j<smallerTile.getTokens().size(); j++) {
TokenEntry smallTileToken = (TokenEntry)smallerTile.getTokens().get(j);
for (int k=0; k<largerTile.getTokens().size(); k++) {
TokenEntry largeTileToken = (TokenEntry)largerTile.getTokens().get(k);
if (smallTileToken.getBeginLine() == largeTileToken.getBeginLine() &&
smallTileToken.getImage().equals(largeTileToken.getImage()) &&
smallTileToken.getTokenSrcID().equals(largeTileToken.getTokenSrcID())) {
super.orderedTiles.remove(smallerTile);
super.tileToOccurrenceMap.remove(smallerTile);
break outer;
}
}
}
}
}
/*
private void removeDupesOf(Tile tile) {
Set occs = new HashSet();
occs.addAll((List)tileToOccurrenceMap.get(tile));
for (Iterator i = tileToOccurrenceMap.keySet().iterator(); i.hasNext();) {
Tile tile2 = (Tile)i.next();
if (tile2.equals(tile)) {
continue;
}
Set possibleDupe = new HashSet();
possibleDupe.addAll((List)tileToOccurrenceMap.get(tile2));
possibleDupe.removeAll(occs);
if (possibleDupe.isEmpty()) {
System.out.println("Removing dupe " + tile.getImage());
tileToOccurrenceMap.remove(tile);
orderedTiles.remove(tile);
break;
}
}
}
*/
}

46
pmd-dcpd/src/net/sourceforge/pmd/dcpd/DGST.java
View File

@ -20,7 +20,7 @@ public class DGST {
private TokenSets tokenSets;
private JavaSpace space;
private Job job;
private Results results = new Results();
private Results results = new DCPDResultsImpl();
public DGST(JavaSpace space, Job job, TokenSets tokenSets, int minimumTileSize) {
this.minimumTileSize = minimumTileSize;
@ -59,39 +59,33 @@ public class DGST {
private void addToResults(Occurrences occ) {
for (Iterator i = occ.getTiles(); i.hasNext();) {
Tile tile = (Tile)i.next();
if (tile.getTokenCount() > this.minimumTileSize) {
// why is this necessary? Seems like Results.clearDupes() or whatever it's called
// should take care of it...
if (!isDuplicate(tile)) {
for (Iterator j = occ.getOccurrences(tile); j.hasNext();) {
TokenEntry te = (TokenEntry)j.next();
results.addTile(tile, te);
}
if (tile.getTokenCount() >= minimumTileSize) {
for (Iterator j = occ.getOccurrences(tile); j.hasNext();) {
TokenEntry te = (TokenEntry)j.next();
results.addTile(tile, te);
}
}
}
}
private boolean isDuplicate(Tile tile) {
/*
private boolean isDuplicate(Tile candidate) {
for (Iterator j = results.getTiles(); j.hasNext();) {
Tile tile2 = (Tile)j.next();
TokenEntry first = (TokenEntry)tile2.getTokens().get(0);
TokenEntry second = (TokenEntry)tile.getTokens().get(0);
if (first.getTokenSrcID().equals(second.getTokenSrcID()) &&
first.getBeginLine() == second.getBeginLine() &&
first.getImage().equals(second.getImage())) {
return true;
Tile tile = (Tile)j.next();
for (int i=0;i<tile.getTokens().size(); i++) {
TokenEntry tok = (TokenEntry)tile.getTokens().get(i);
for (int k=0; k<candidate.getTokens().size(); k++) {
TokenEntry candidateToken = (TokenEntry)candidate.getTokens().get(k);
if (tok.getTokenSrcID().equals(candidateToken.getTokenSrcID()) &&
tok.getBeginLine() == candidateToken.getBeginLine() &&
tok.getImage().equals(candidateToken.getImage())) {
System.out.println("DISCARD");
return true;
}
}
}
}
return false;
}
private List marshal(Iterator i) {
List list = new ArrayList();
while (i.hasNext()) {
list.add(i.next());
}
return list;
}
*/
}

61
pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileExpander.java
View File

@ -28,67 +28,44 @@ public class TileExpander {
}
public void expandAvailableTiles() throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{
Entry twQuery = space.snapshot(new TileWrapper(null, null, tsw.jobID, TileWrapper.NOT_DONE, null, null, null, null));
Entry twQuery = space.snapshot(new Chunk(tsw.jobID, null, Chunk.NOT_DONE, null));
TileWrapper tileWrapperToExpand = null;
Chunk chunk = null;
int total = 0;
while ((tileWrapperToExpand = (TileWrapper)space.take(twQuery, null, 10)) != null) {
while ((chunk = (Chunk)space.take(twQuery, null, 10)) != null) {
total++;
//System.out.println("Expanding " + tileWrapperToExpand.tile.getImage());
Occurrences results = expand(tileWrapperToExpand);
int expansionIndex = 0;
for (Iterator i = results.getTiles();i.hasNext();) {
Tile tile = (Tile)i.next();
TileWrapper tileWrapperToWrite = new TileWrapper(tile,
marshal(results.getOccurrences(tile)),
tsw.jobID,
TileWrapper.DONE,
null,
tileWrapperToExpand.originalTilePosition,
new Integer(expansionIndex),
new Integer(results.size()));
space.write(tileWrapperToWrite, null, Lease.FOREVER);
//System.out.println("Wrote " + tileWrapperToWrite + "; occurrences = " + tileWrapperToWrite.occurrences.size());
expansionIndex++;
List wrappers = new ArrayList();
for (int j=0; j<chunk.tileWrappers.size(); j++) {
TileWrapper tileWrapperToExpand = (TileWrapper)chunk.tileWrappers.get(j);
//System.out.println("Expanding " + tileWrapperToExpand.tile.getImage());
Occurrences results = expand(tileWrapperToExpand, j);
int expansionIndex = 0;
for (Iterator i = results.getTiles();i.hasNext();) {
Tile tile = (Tile)i.next();
TileWrapper tileWrapperToWrite = new TileWrapper(tile, results.getOccurrencesList(tile), new Integer(expansionIndex), new Integer(results.size()));
wrappers.add(tileWrapperToWrite);
//System.out.println("Wrote " + tileWrapperToWrite + "; occurrences = " + tileWrapperToWrite.occurrences.size());
expansionIndex++;
}
}
Chunk chunkToWrite = new Chunk(tsw.jobID, wrappers, Chunk.DONE, chunk.sequenceID);
space.write(chunkToWrite, null, Lease.FOREVER);
}
if (total>0) System.out.println("Expanded " + total + " tiles");
}
private List marshal(Iterator i) {
List list = new ArrayList();
while (i.hasNext()) {
list.add(i.next());
}
return list;
}
private Occurrences expand(TileWrapper tileWrapper) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{
private Occurrences expand(TileWrapper tileWrapper, int sequenceID) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException{
Occurrences newOcc = new Occurrences(new CPDNullListener());
for (Iterator i = tileWrapper.occurrences.iterator(); i.hasNext();) {
TokenEntry tok = (TokenEntry)i.next();
TokenList tokenSet = tsw.tokenSets.getTokenList(tok);
if (tokenSet.hasTokenAfter(tileWrapper.tile, tok)) {
TokenEntry token = (TokenEntry)tokenSet.get(tok.getIndex() + tileWrapper.tile.getTokenCount());
// make sure the next token hasn't already been used in an occurrence
if (!newOcc.contains(token)) {
Tile newTile = tileWrapper.tile.copy();
newTile.add(token);
newOcc.addTile(newTile, tok);
}
} else {
// we have to put something back in the space to let DGST know that
// this tile has been processed...
TileWrapper tileWrapperToWrite = new TileWrapper(tileWrapper.tile,
new ArrayList(),
tsw.jobID,
TileWrapper.DONE,
TileWrapper.DISCARD_DUE_TO_EOF,
tileWrapper.originalTilePosition,
new Integer(0),
null);
space.write(tileWrapperToWrite, null, Lease.FOREVER);
}
}
return newOcc;

34
pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileHarvester.java
View File

@ -29,42 +29,16 @@ public class TileHarvester {
public Occurrences harvest(int originalOccurrencesCount) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException {
Occurrences occ = new Occurrences(new CPDNullListener());
for (int i=0;i<originalOccurrencesCount; i++) {
TileWrapper tw = (TileWrapper)space.take(new TileWrapper(null,
null,
job.id,
TileWrapper.DONE,
null,
new Integer(i), new Integer(0), null), null, Lease.FOREVER);
if (tw.discardFlag == null) {
addAllExpansions(i, tw, occ);
Chunk chunk = (Chunk)space.take(new Chunk(job.id, null, Chunk.DONE, new Integer(i)), null, Lease.FOREVER);
for (int j=0; j<chunk.tileWrappers.size(); j++) {
addTileWrapperToOccurrences((TileWrapper)chunk.tileWrappers.get(j), occ);
}
}
return occ;
}
private void addAllExpansions(int originalPosition, TileWrapper firstExpansion, Occurrences occ) throws RemoteException, UnusableEntryException, TransactionException, InterruptedException {
TileWrapper nextExpansion = firstExpansion;
for (int i=0; i<firstExpansion.expansionsTotal.intValue(); i++) {
if (i>0) {
nextExpansion = (TileWrapper)space.take(new TileWrapper(null,
null,
job.id,
TileWrapper.DONE,
null,
new Integer(originalPosition),
new Integer(i),
firstExpansion.expansionsTotal), null, Lease.FOREVER);
}
//System.out.println("Gathered " + nextExpansion + "; occurrences = " + nextExpansion.occurrences.size());
// here's where we discard solo tiles
if (nextExpansion.occurrences.size() > 1) {
addTileWrapperToOccurrences(nextExpansion, occ);
}
}
}
private void addTileWrapperToOccurrences(TileWrapper tw, Occurrences occ) {
if (!occ.containsAnyTokensIn(tw.tile)) {
if (tw.occurrences.size() > 1 && !occ.containsAnyTokensIn(tw.tile)) {
for (int i=0; i<tw.occurrences.size(); i++) {
occ.addTile(tw.tile, (TokenEntry)tw.occurrences.get(i));
}

22
pmd-dcpd/src/net/sourceforge/pmd/dcpd/TilePlanter.java
View File

@ -30,14 +30,11 @@ public class TilePlanter {
int tilesSoFar=0;
for (Iterator i = occ.getTiles(); i.hasNext();) {
Tile tile = (Tile)i.next();
TileWrapper tw = new TileWrapper(tile,
marshal(occ.getOccurrences(tile)),
job.id,
TileWrapper.NOT_DONE,
null,
new Integer(tilesSoFar),
null, null);
space.write(tw, null, Lease.FOREVER);
TileWrapper tw = new TileWrapper(tile, occ.getOccurrencesList(tile), null, null);
List wrappers = new ArrayList();
wrappers.add(tw);
Chunk chunk = new Chunk(job.id, wrappers, Chunk.NOT_DONE, new Integer(tilesSoFar));
space.write(chunk, null, Lease.FOREVER);
//System.out.println("Scattering " + tw.tile.getImage() + "->" + tw.occurrences.size());
tilesSoFar++;
if (tilesSoFar % 100 == 0) {
@ -45,13 +42,4 @@ public class TilePlanter {
}
}
}
private List marshal(Iterator i) {
List list = new ArrayList();
while (i.hasNext()) {
list.add(i.next());
}
return list;
}
}

21
pmd-dcpd/src/net/sourceforge/pmd/dcpd/TileWrapper.java
View File

@ -12,40 +12,25 @@ import java.util.List;
public class TileWrapper implements Entry {
// routing information
public static final Integer DONE = new Integer( 1 );
public static final Integer NOT_DONE = new Integer( 0 );
public static final Integer DISCARD_DUE_TO_EOF = new Integer(0);
public Integer isDone;
public Integer discardFlag;
// payload information
public Tile tile;
public Integer jobID;
public List occurrences;
public Integer originalTilePosition;
public Integer expansionIndex;
public Integer expansionsTotal;
public TileWrapper() {}
public TileWrapper(Tile tile, List occurrences, Integer jobID, Integer isDone, Integer discardFlag, Integer originalTilePosition, Integer expansionIndex, Integer expansionsTotal) {
public TileWrapper(Tile tile, List occurrences, Integer expansionIndex, Integer expansionsTotal) {
this.tile = tile;
this.jobID = jobID;
this.occurrences = occurrences;
this.isDone = isDone;
this.discardFlag = discardFlag;
this.originalTilePosition = originalTilePosition;
this.expansionIndex = expansionIndex;
this.expansionsTotal = expansionsTotal;
}
public String toString() {
return tile.getImage() + ":" + isDone + ":" + getExpansionIndexPicture();
return tile.getImage() + ":" + getExpansionIndexPicture();
}
public String getExpansionIndexPicture() {
return "(" + originalTilePosition + ":" + expansionIndex + "/" + expansionsTotal + ")";
return "(" + expansionIndex + "/" + expansionsTotal + ")";
}
}