From bc18c1280a157cba8a8e7f07b509afbd4a4e44ec Mon Sep 17 00:00:00 2001
From: Tom Copeland <tomcopeland@users.sourceforge.net>
Date: Mon, 23 Sep 2002 20:19:33 +0000
Subject: [PATCH] added a new tokenizer

git-svn-id: https://pmd.svn.sourceforge.net/svnroot/pmd/trunk@974 51baf565-9d33-0410-a72c-fc3788e3496d
---
 pmd/src/net/sourceforge/pmd/cpd/CPD.java      | 52 ++++++++++---------
 .../pmd/cpd/JavaTokensTokenizer.java          |  4 --
 .../sourceforge/pmd/cpd/LinesTokenizer.java   | 34 ++++++++++++
 3 files changed, 61 insertions(+), 29 deletions(-)
 create mode 100644 pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java

diff --git a/pmd/src/net/sourceforge/pmd/cpd/CPD.java b/pmd/src/net/sourceforge/pmd/cpd/CPD.java
index bf73398009..91aafebc38 100644
--- a/pmd/src/net/sourceforge/pmd/cpd/CPD.java
+++ b/pmd/src/net/sourceforge/pmd/cpd/CPD.java
@@ -29,16 +29,6 @@ public class CPD {
         }
     }
 
-    public void add(int fileCount, File file) throws IOException {
-        listener.addedFile(fileCount, file);
-        Tokenizer t = new JavaTokensTokenizer();
-        TokenList ts = new TokenList(file.getAbsolutePath());
-        FileReader fr = new FileReader(file);
-        t.tokenize(ts, fr);
-        fr.close();
-        tokenSets.add(ts);
-    }
-
     public void setMinimumTileSize(int tileSize) {
         minimumTileSize = tileSize;
     }
@@ -55,20 +45,6 @@ public class CPD {
         addDirectory(dir, true);
     }
 
-    private void addDirectory(String dir, boolean recurse) throws IOException {
-        FileFinder finder = new FileFinder();
-        List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
-        add(list);
-    }
-
-    public void add(String id, String input) throws IOException {
-        Tokenizer t = new JavaTokensTokenizer();
-        TokenList ts = new TokenList(id);
-        t.tokenize(ts, new StringReader(input));
-        tokenSets.add(ts);
-    }
-
-
     public void go() {
         if (!listener.update("Starting to process " + tokenSets.size() + " files")) return;
         GST gst = new GST(tokenSets, minimumTileSize);
@@ -96,6 +72,30 @@ public class CPD {
         return "";
     }
 
+    private void addDirectory(String dir, boolean recurse) throws IOException {
+        FileFinder finder = new FileFinder();
+        List list = finder.findFilesFrom(dir, new JavaFileOrDirectoryFilter(), recurse);
+        add(list);
+    }
+
+    public void add(String id, String input) throws IOException {
+        Tokenizer t = new JavaTokensTokenizer();
+        TokenList ts = new TokenList(id);
+        t.tokenize(ts, new StringReader(input));
+        tokenSets.add(ts);
+    }
+
+    private void add(int fileCount, File file) throws IOException {
+        listener.addedFile(fileCount, file);
+        Tokenizer t = new JavaTokensTokenizer();
+        //Tokenizer t = new LinesTokenizer();
+        TokenList ts = new TokenList(file.getAbsolutePath());
+        FileReader fr = new FileReader(file);
+        t.tokenize(ts, fr);
+        fr.close();
+        tokenSets.add(ts);
+    }
+
     public static void main(String[] args) {
         CPD cpd = new CPD();
         cpd.setListener(new CPDNullListener());
@@ -108,9 +108,11 @@ public class CPD {
         }
         long start = System.currentTimeMillis();
         cpd.go();
-        System.out.println("That took " + (System.currentTimeMillis() - start));
+        long total = System.currentTimeMillis() - start;
+        System.out.println("That took " + total);
         CPDRenderer renderer = new TextRenderer();
         System.out.println(renderer.render(cpd));
+        System.out.println("That took " + total);
     }
 
 }
\ No newline at end of file
diff --git a/pmd/src/net/sourceforge/pmd/cpd/JavaTokensTokenizer.java b/pmd/src/net/sourceforge/pmd/cpd/JavaTokensTokenizer.java
index 05bffffdca..ea9350c7b6 100644
--- a/pmd/src/net/sourceforge/pmd/cpd/JavaTokensTokenizer.java
+++ b/pmd/src/net/sourceforge/pmd/cpd/JavaTokensTokenizer.java
@@ -18,10 +18,6 @@ import java.util.ArrayList;
 public class JavaTokensTokenizer implements Tokenizer {
 
     private boolean discarding;
-
-    /**
-     * The end of line string for this machine.
-     */
     protected String EOL = System.getProperty("line.separator", "\n");
 
     public void tokenize(TokenList tokens, Reader input) throws IOException {
diff --git a/pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java b/pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java
new file mode 100644
index 0000000000..fc335132d8
--- /dev/null
+++ b/pmd/src/net/sourceforge/pmd/cpd/LinesTokenizer.java
@@ -0,0 +1,34 @@
+/*
+ * User: tom
+ * Date: Sep 23, 2002
+ * Time: 3:14:38 PM
+ */
+package net.sourceforge.pmd.cpd;
+
+import net.sourceforge.pmd.ast.JavaCharStream;
+import net.sourceforge.pmd.ast.JavaParserTokenManager;
+
+import java.io.Reader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.StringReader;
+import java.util.List;
+import java.util.ArrayList;
+
+public class LinesTokenizer implements Tokenizer {
+
+    public void tokenize(TokenList tokens, Reader input) throws IOException {
+        String eol = System.getProperty("line.separator", "\n");
+        List lines = new ArrayList();
+        StringBuffer sb = new StringBuffer();
+        LineNumberReader r = new LineNumberReader(input);
+        String currentLine = null;
+        while ((currentLine = r.readLine()) != null) {
+            lines.add(currentLine);
+            sb.append(currentLine);
+            sb.append(eol);
+            tokens.add(new TokenEntry(currentLine, tokens.size(), tokens.getID(), lines.size()));
+        }
+        tokens.setCode(lines);
+    }
+}