Modified C++ tokenizer to use the JavaCC STATIC option; this results in about a 30% speedup in tokenizing. Also improved error messages.

git-svn-id: https://pmd.svn.sourceforge.net/svnroot/pmd/trunk@4486 51baf565-9d33-0410-a72c-fc3788e3496d

pmd/bin/build.xml

@@ -185,10 +185,7 @@
 		<delete>
 			<fileset dir="src/net/sourceforge/pmd/cpd/cppast">
 				<include name="*.java" />
-				<exclude name="ClassScope.java" />
-				<exclude name="Declaration.java" />
-				<exclude name="Scope.java" />
-				<exclude name="SymtabManager.java" />
+                <exclude name="TokenMgrError.java" /> <!-- TokenMgrError customized to report filename, so don't delete it -->
 			</fileset>
 		</delete>
 		<javacc target="etc/grammar/cpp.jj" outputdirectory="src/net/sourceforge/pmd/cpd/cppast" javacchome="${javacc-home.path}" />

pmd/etc/changelog.txt

@@ -14,9 +14,10 @@ Fixed bug 1522054 - BooleanInstantiation now detects instantiations inside metho
 Fixed bug 1522056 - UseStringBufferForStringAppends now flags appends which occur in static initializers and constructors
 Fixed bug 1526530 - SingularField now finds fields which are hidden at the method or static level
 Fixed bug 1529805 - UnusedModifier no longer throws NPEs on JDK 1.5 enums.
-Fixed bug 1531593 - UnnecessaryConversionTemporary no longer reports false positives when toString() is invoked inside the call to 'new Long/Integer/etc()'.  
+Fixed bug 1531593 - UnnecessaryConversionTemporary no longer reports false positives when toString() is invoked inside the call to 'new Long/Integer/etc()'.
 Fixed a bug in AvoidProtectedFieldInFinalClass - it no longer reports false positives for protected fields in inner classes.
 Fixed a bug in the C++ grammar - the tokenizer now properly recognizes macro definitions which are followed by a multiline comment.
+Modified C++ tokenizer to use the JavaCC STATIC option; this results in about a 30% speedup in tokenizing.  Also improved error messages.
 Implemented RFE 1501850 - UnusedFormalParameter now catches cases where a parameter is assigned to but not used.
 Applied patch 1481024 (implementing RFE 1490181)- NOPMD messages can now be reported with a user specified msg, e.g., //NOPMD - this is expected
 Added JSP support to the copy/paste detector.

pmd/etc/grammar/cpp.jj

@@ -24,7 +24,6 @@
  *
  */
 options {
-  STATIC=false;
   BUILD_PARSER=false;
   CACHE_TOKENS=true;
 }
@@ -129,6 +128,16 @@ public final class CPPParser {
 
 PARSER_END(CPPParser)
 
+TOKEN_MGR_DECLS : {
+ private static String filename;
+ public static void setFileName(String name) {
+  filename = name;
+ }
+ public static String getFileName() {
+  return filename;
+ }
+}
+
 SKIP :
 {
   " "

pmd/src/net/sourceforge/pmd/RuleSet.java

@@ -98,42 +98,6 @@ public class RuleSet {
         }
     }
 
-    /**
-     * Gives the name of this ruleset
-     *
-     * @return a String representing the name
-     */
-    public String getName() {
-        return name;
-    }
-
-    /**
-     * Set the name of this ruleset
-     *
-     * @param name a String representing the name
-     */
-    public void setName(String name) {
-        this.name = name;
-    }
-
-    /**
-     * Gives the description of this ruleset
-     *
-     * @return a String representing the description
-     */
-    public String getDescription() {
-        return description;
-    }
-
-    /**
-     * Set the description of this ruleset
-     *
-     * @param description a String representing the description
-     */
-    public void setDescription(String description) {
-        this.description = description;
-    }
-
     /**
      * @see java.lang.Object#equals(java.lang.Object)
      */
@@ -157,17 +121,28 @@ public class RuleSet {
         return this.getName().hashCode() + 13 * this.getRules().hashCode();
     }
 
-    /**
-     * @return Returns the language.
-     */
     public Language getLanguage() {
         return language;
     }
 
-    /**
-     * @param language The language to set.
-     */
     public void setLanguage(Language language) {
         this.language = language;
     }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
 }

pmd/src/net/sourceforge/pmd/RuleSets.java

@@ -54,6 +54,10 @@ public class RuleSets {
         return (RuleSet[]) ruleSets.toArray(new RuleSet[0]);
     }
 
+    public Iterator getRuleSetsIterator() {
+        return ruleSets.iterator();
+    }
+
     /**
      * Return all rules from all rulesets.
      *

pmd/src/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -15,14 +15,28 @@ import java.util.List;
 public class CPPTokenizer implements Tokenizer {
     protected String EOL = System.getProperty("line.separator", "\n");
 
+    private static SimpleCharStream charStream;
     public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
         StringBuffer sb = sourceCode.getCodeBuffer();
         try {
-            CPPParserTokenManager tokenManager = new CPPParserTokenManager(new SimpleCharStream(new StringReader(sb.toString())));
-            Token currToken = tokenManager.getNextToken();
+/*
+            if (c == null) {
+                c = new SimpleCharStream(new StringReader(sb.toString()));
+            } else {
+                c.ReInit(new StringReader(sb.toString()));
+            }
+*/
+            if (charStream == null) {
+                charStream = new SimpleCharStream(new StringReader(sb.toString()));
+            } else {
+                charStream.ReInit(new StringReader(sb.toString()));
+            }
+            CPPParserTokenManager.ReInit(charStream);
+            CPPParserTokenManager.setFileName(sourceCode.getFileName());
+            Token currToken = CPPParserTokenManager.getNextToken();
             while (currToken.image.length() > 0) {
                 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
-                currToken = tokenManager.getNextToken();
+                currToken = CPPParserTokenManager.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.out.println("Added " + sourceCode.getFileName());

pmd/src/net/sourceforge/pmd/cpd/cppast/SimpleCharStream.java

@@ -8,32 +8,32 @@ package net.sourceforge.pmd.cpd.cppast;
 
 public class SimpleCharStream
 {
-  public static final boolean staticFlag = false;
-  int bufsize;
-  int available;
-  int tokenBegin;
-  public int bufpos = -1;
-  protected int bufline[];
-  protected int bufcolumn[];
+  public static final boolean staticFlag = true;
+  static int bufsize;
+  static int available;
+  static int tokenBegin;
+  static public int bufpos = -1;
+  static protected int bufline[];
+  static protected int bufcolumn[];
 
-  protected int column = 0;
-  protected int line = 1;
+  static protected int column = 0;
+  static protected int line = 1;
 
-  protected boolean prevCharIsCR = false;
-  protected boolean prevCharIsLF = false;
+  static protected boolean prevCharIsCR = false;
+  static protected boolean prevCharIsLF = false;
 
-  protected java.io.Reader inputStream;
+  static protected java.io.Reader inputStream;
 
-  protected char[] buffer;
-  protected int maxNextCharInd = 0;
-  protected int inBuf = 0;
-  protected int tabSize = 8;
+  static protected char[] buffer;
+  static protected int maxNextCharInd = 0;
+  static protected int inBuf = 0;
+  static protected int tabSize = 8;
 
-  protected void setTabSize(int i) { tabSize = i; }
-  protected int getTabSize(int i) { return tabSize; }
+  static protected void setTabSize(int i) { tabSize = i; }
+  static protected int getTabSize(int i) { return tabSize; }
 
 
-  protected void ExpandBuff(boolean wrapAround)
+  static protected void ExpandBuff(boolean wrapAround)
   {
      char[] newbuffer = new char[bufsize + 2048];
      int newbufline[] = new int[bufsize + 2048];
@@ -83,7 +83,7 @@ public class SimpleCharStream
      tokenBegin = 0;
   }
 
-  protected void FillBuff() throws java.io.IOException
+  static protected void FillBuff() throws java.io.IOException
   {
      if (maxNextCharInd == available)
      {
@@ -128,7 +128,7 @@ public class SimpleCharStream
      }
   }
 
-  public char BeginToken() throws java.io.IOException
+  static public char BeginToken() throws java.io.IOException
   {
      tokenBegin = -1;
      char c = readChar();
@@ -137,7 +137,7 @@ public class SimpleCharStream
      return c;
   }
 
-  protected void UpdateLineColumn(char c)
+  static protected void UpdateLineColumn(char c)
   {
      column++;
 
@@ -177,7 +177,7 @@ public class SimpleCharStream
      bufcolumn[bufpos] = column;
   }
 
-  public char readChar() throws java.io.IOException
+  static public char readChar() throws java.io.IOException
   {
      if (inBuf > 0)
      {
@@ -203,7 +203,7 @@ public class SimpleCharStream
    * @see #getEndColumn
    */
 
-  public int getColumn() {
+  static public int getColumn() {
      return bufcolumn[bufpos];
   }
 
@@ -212,27 +212,27 @@ public class SimpleCharStream
    * @see #getEndLine
    */
 
-  public int getLine() {
+  static public int getLine() {
      return bufline[bufpos];
   }
 
-  public int getEndColumn() {
+  static public int getEndColumn() {
      return bufcolumn[bufpos];
   }
 
-  public int getEndLine() {
+  static public int getEndLine() {
      return bufline[bufpos];
   }
 
-  public int getBeginColumn() {
+  static public int getBeginColumn() {
      return bufcolumn[tokenBegin];
   }
 
-  public int getBeginLine() {
+  static public int getBeginLine() {
      return bufline[tokenBegin];
   }
 
-  public void backup(int amount) {
+  static public void backup(int amount) {
 
     inBuf += amount;
     if ((bufpos -= amount) < 0)
@@ -242,6 +242,10 @@ public class SimpleCharStream
   public SimpleCharStream(java.io.Reader dstream, int startline,
   int startcolumn, int buffersize)
   {
+    if (inputStream != null)
+       throw new Error("\n   ERROR: Second call to the constructor of a static SimpleCharStream.  You must\n" +
+       "       either use ReInit() or set the JavaCC option STATIC to false\n" +
+       "       during the generation of this class.");
     inputStream = dstream;
     line = startline;
     column = startcolumn - 1;
@@ -356,7 +360,7 @@ public class SimpleCharStream
   {
      ReInit(dstream, startline, startcolumn, 4096);
   }
-  public String GetImage()
+  static public String GetImage()
   {
      if (bufpos >= tokenBegin)
         return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
@@ -365,7 +369,7 @@ public class SimpleCharStream
                               new String(buffer, 0, bufpos + 1);
   }
 
-  public char[] GetSuffix(int len)
+  static public char[] GetSuffix(int len)
   {
      char[] ret = new char[len];
 
@@ -381,7 +385,7 @@ public class SimpleCharStream
      return ret;
   }
 
-  public void Done()
+  static public void Done()
   {
      buffer = null;
      bufline = null;
@@ -391,7 +395,7 @@ public class SimpleCharStream
   /**
    * Method to adjust line and column numbers for the start of a token.
    */
-  public void adjustBeginLineColumn(int newLine, int newCol)
+  static public void adjustBeginLineColumn(int newLine, int newCol)
   {
      int start = tokenBegin;
      int len;

pmd/src/net/sourceforge/pmd/cpd/cppast/TokenMgrError.java

@@ -95,7 +95,7 @@ public class TokenMgrError extends Error
     * Note: You can customize the lexical error message by modifying this method.
     */
    protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
-      return("Lexical error at line " +
+      return("Lexical error in file " + CPPParserTokenManager.getFileName() + " at line " +
            errorLine + ", column " +
            errorColumn + ".  Encountered: " +
            (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +