added a new tokenizer
git-svn-id: https://pmd.svn.sourceforge.net/svnroot/pmd/trunk@974 51baf565-9d33-0410-a72c-fc3788e3496d
This commit is contained in:
pmd/src/net/sourceforge/pmd/cpd
@ -29,16 +29,6 @@ public class CPD {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void add(int fileCount, File file) throws IOException {
|
|
||||||
listener.addedFile(fileCount, file);
|
|
||||||
Tokenizer t = new JavaTokensTokenizer();
|
|
||||||
TokenList ts = new TokenList(file.getAbsolutePath());
|
|
||||||
FileReader fr = new FileReader(file);
|
|
||||||
t.tokenize(ts, fr);
|
|
||||||
fr.close();
|
|
||||||
tokenSets.add(ts);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setMinimumTileSize(int tileSize) {
|
public void setMinimumTileSize(int tileSize) {
|
||||||
minimumTileSize = tileSize;
|
minimumTileSize = tileSize;
|
||||||
}
|
}
|
||||||
@ -55,20 +45,6 @@ public class CPD {
|
|||||||
addDirectory(dir, true);
|
addDirectory(dir, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addDirectory(String dir, boolean recurse) throws IOException {
|
|
||||||
FileFinder finder = new FileFinder();
|
|
||||||
List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
|
|
||||||
add(list);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void add(String id, String input) throws IOException {
|
|
||||||
Tokenizer t = new JavaTokensTokenizer();
|
|
||||||
TokenList ts = new TokenList(id);
|
|
||||||
t.tokenize(ts, new StringReader(input));
|
|
||||||
tokenSets.add(ts);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void go() {
|
public void go() {
|
||||||
if (!listener.update("Starting to process " + tokenSets.size() + " files")) return;
|
if (!listener.update("Starting to process " + tokenSets.size() + " files")) return;
|
||||||
GST gst = new GST(tokenSets, minimumTileSize);
|
GST gst = new GST(tokenSets, minimumTileSize);
|
||||||
@ -96,6 +72,30 @@ public class CPD {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void addDirectory(String dir, boolean recurse) throws IOException {
|
||||||
|
FileFinder finder = new FileFinder();
|
||||||
|
List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
|
||||||
|
add(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void add(String id, String input) throws IOException {
|
||||||
|
Tokenizer t = new JavaTokensTokenizer();
|
||||||
|
TokenList ts = new TokenList(id);
|
||||||
|
t.tokenize(ts, new StringReader(input));
|
||||||
|
tokenSets.add(ts);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void add(int fileCount, File file) throws IOException {
|
||||||
|
listener.addedFile(fileCount, file);
|
||||||
|
Tokenizer t = new JavaTokensTokenizer();
|
||||||
|
//Tokenizer t = new LinesTokenizer();
|
||||||
|
TokenList ts = new TokenList(file.getAbsolutePath());
|
||||||
|
FileReader fr = new FileReader(file);
|
||||||
|
t.tokenize(ts, fr);
|
||||||
|
fr.close();
|
||||||
|
tokenSets.add(ts);
|
||||||
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
CPD cpd = new CPD();
|
CPD cpd = new CPD();
|
||||||
cpd.setListener(new CPDNullListener());
|
cpd.setListener(new CPDNullListener());
|
||||||
@ -108,9 +108,11 @@ public class CPD {
|
|||||||
}
|
}
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
cpd.go();
|
cpd.go();
|
||||||
System.out.println("That took " + (System.currentTimeMillis() - start));
|
long total = System.currentTimeMillis() - start;
|
||||||
|
System.out.println("That took " + total);
|
||||||
CPDRenderer renderer = new TextRenderer();
|
CPDRenderer renderer = new TextRenderer();
|
||||||
System.out.println(renderer.render(cpd));
|
System.out.println(renderer.render(cpd));
|
||||||
|
System.out.println("That took " + total);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
@ -18,10 +18,6 @@ import java.util.ArrayList;
|
|||||||
public class JavaTokensTokenizer implements Tokenizer {
|
public class JavaTokensTokenizer implements Tokenizer {
|
||||||
|
|
||||||
private boolean discarding;
|
private boolean discarding;
|
||||||
|
|
||||||
/**
|
|
||||||
* The end of line string for this machine.
|
|
||||||
*/
|
|
||||||
protected String EOL = System.getProperty("line.separator", "\n");
|
protected String EOL = System.getProperty("line.separator", "\n");
|
||||||
|
|
||||||
public void tokenize(TokenList tokens, Reader input) throws IOException {
|
public void tokenize(TokenList tokens, Reader input) throws IOException {
|
||||||
|
34
pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java
Normal file
34
pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
/*
|
||||||
|
* User: tom
|
||||||
|
* Date: Sep 23, 2002
|
||||||
|
* Time: 3:14:38 PM
|
||||||
|
*/
|
||||||
|
package net.sourceforge.pmd.cpd;
|
||||||
|
|
||||||
|
import net.sourceforge.pmd.ast.JavaCharStream;
|
||||||
|
import net.sourceforge.pmd.ast.JavaParserTokenManager;
|
||||||
|
|
||||||
|
import java.io.Reader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.LineNumberReader;
|
||||||
|
import java.io.StringReader;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
public class LinesTokenizer implements Tokenizer {
|
||||||
|
|
||||||
|
public void tokenize(TokenList tokens, Reader input) throws IOException {
|
||||||
|
String eol = System.getProperty("line.separator", "\n");
|
||||||
|
List lines = new ArrayList();
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
LineNumberReader r = new LineNumberReader(input);
|
||||||
|
String currentLine = null;
|
||||||
|
while ((currentLine = r.readLine()) != null) {
|
||||||
|
lines.add(currentLine);
|
||||||
|
sb.append(currentLine);
|
||||||
|
sb.append(eol);
|
||||||
|
tokens.add(new TokenEntry(currentLine, tokens.size(), tokens.getID(), lines.size()));
|
||||||
|
}
|
||||||
|
tokens.setCode(lines);
|
||||||
|
}
|
||||||
|
}
|
Reference in New Issue
Block a user