added a new tokenizer

git-svn-id: https://pmd.svn.sourceforge.net/svnroot/pmd/trunk@974 51baf565-9d33-0410-a72c-fc3788e3496d

pmd/src/net/sourceforge/pmd/cpd/CPD.java

@@ -29,16 +29,6 @@ public class CPD {
         }
     }
 
-    public void add(int fileCount, File file) throws IOException {
-        listener.addedFile(fileCount, file);
-        Tokenizer t = new JavaTokensTokenizer();
-        TokenList ts = new TokenList(file.getAbsolutePath());
-        FileReader fr = new FileReader(file);
-        t.tokenize(ts, fr);
-        fr.close();
-        tokenSets.add(ts);
-    }
-
     public void setMinimumTileSize(int tileSize) {
         minimumTileSize = tileSize;
     }
@@ -55,20 +45,6 @@ public class CPD {
         addDirectory(dir, true);
     }
 
-    private void addDirectory(String dir, boolean recurse) throws IOException {
-        FileFinder finder = new FileFinder();
-        List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
-        add(list);
-    }
-
-    public void add(String id, String input) throws IOException {
-        Tokenizer t = new JavaTokensTokenizer();
-        TokenList ts = new TokenList(id);
-        t.tokenize(ts, new StringReader(input));
-        tokenSets.add(ts);
-    }
-
-
     public void go() {
         if (!listener.update("Starting to process " + tokenSets.size() + " files")) return;
         GST gst = new GST(tokenSets, minimumTileSize);
@@ -96,6 +72,30 @@ public class CPD {
         return "";
     }
 
+    private void addDirectory(String dir, boolean recurse) throws IOException {
+        FileFinder finder = new FileFinder();
+        List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
+        add(list);
+    }
+
+    public void add(String id, String input) throws IOException {
+        Tokenizer t = new JavaTokensTokenizer();
+        TokenList ts = new TokenList(id);
+        t.tokenize(ts, new StringReader(input));
+        tokenSets.add(ts);
+    }
+
+    private void add(int fileCount, File file) throws IOException {
+        listener.addedFile(fileCount, file);
+        Tokenizer t = new JavaTokensTokenizer();
+        //Tokenizer t = new LinesTokenizer();
+        TokenList ts = new TokenList(file.getAbsolutePath());
+        FileReader fr = new FileReader(file);
+        t.tokenize(ts, fr);
+        fr.close();
+        tokenSets.add(ts);
+    }
+
     public static void main(String[] args) {
         CPD cpd = new CPD();
         cpd.setListener(new CPDNullListener());
@@ -108,9 +108,11 @@ public class CPD {
         }
         long start = System.currentTimeMillis();
         cpd.go();
-        System.out.println("That took " + (System.currentTimeMillis() - start));
+        long total = System.currentTimeMillis() - start;
+        System.out.println("That took " + total);
         CPDRenderer renderer = new TextRenderer();
         System.out.println(renderer.render(cpd));
+        System.out.println("That took " + total);
     }
 
 }

pmd/src/net/sourceforge/pmd/cpd/JavaTokensTokenizer.java

@@ -18,10 +18,6 @@ import java.util.ArrayList;
 public class JavaTokensTokenizer implements Tokenizer {
 
     private boolean discarding;
-
-    /**
-     * The end of line string for this machine.
-     */
     protected String EOL = System.getProperty("line.separator", "\n");
 
     public void tokenize(TokenList tokens, Reader input) throws IOException {

pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java

@@ -0,0 +1,34 @@
+/*
+ * User: tom
+ * Date: Sep 23, 2002
+ * Time: 3:14:38 PM
+ */
+package net.sourceforge.pmd.cpd;
+
+import net.sourceforge.pmd.ast.JavaCharStream;
+import net.sourceforge.pmd.ast.JavaParserTokenManager;
+
+import java.io.Reader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.StringReader;
+import java.util.List;
+import java.util.ArrayList;
+
+public class LinesTokenizer implements Tokenizer {
+
+    public void tokenize(TokenList tokens, Reader input) throws IOException {
+        String eol = System.getProperty("line.separator", "\n");
+        List lines = new ArrayList();
+        StringBuffer sb = new StringBuffer();
+        LineNumberReader r = new LineNumberReader(input);
+        String currentLine = null;
+        while ((currentLine = r.readLine()) != null) {
+            lines.add(currentLine);
+            sb.append(currentLine);
+            sb.append(eol);
+            tokens.add(new TokenEntry(currentLine, tokens.size(), tokens.getID(), lines.size()));
+        }
+        tokens.setCode(lines);
+    }
+}