Merge branch 'cpd-no-ignore-constructor' of https://github.com/Monits/pmd into pr-106

pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java

@@ -4,6 +4,8 @@
 package net.sourceforge.pmd.cpd;
 
 import java.io.StringReader;
+import java.util.Deque;
+import java.util.LinkedList;
 import java.util.Properties;
 
 import net.sourceforge.pmd.lang.LanguageRegistry;
@@ -21,7 +23,7 @@ public class JavaTokenizer implements Tokenizer {
     private boolean ignoreAnnotations;
     private boolean ignoreLiterals;
     private boolean ignoreIdentifiers;
-
+    
     public void setProperties(Properties properties) {
         ignoreAnnotations = Boolean.parseBoolean(properties.getProperty(IGNORE_ANNOTATIONS, "false"));
         ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
@@ -39,6 +41,7 @@ public class JavaTokenizer implements Tokenizer {
         Token currentToken = (Token) tokenMgr.getNextToken();
 
         TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
+        ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
 
         while (currentToken.image.length() > 0) {
             discarder.updateState(currentToken);
@@ -48,14 +51,17 @@ public class JavaTokenizer implements Tokenizer {
                 continue;
             }
 
-            processToken(tokenEntries, fileName, currentToken);
+            processToken(tokenEntries, fileName, currentToken, constructorDetector);
             currentToken = (Token) tokenMgr.getNextToken();
         }
         tokenEntries.add(TokenEntry.getEOF());
     }
 
-    private void processToken(Tokens tokenEntries, String fileName, Token currentToken) {
+    private void processToken(Tokens tokenEntries, String fileName, Token currentToken, ConstructorDetector constructorDetector) {
         String image = currentToken.image;
+        
+        constructorDetector.restoreConstructorToken(tokenEntries, currentToken);
+        
         if (ignoreLiterals
                 && (currentToken.kind == JavaParserConstants.STRING_LITERAL
                 || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
@@ -66,6 +72,9 @@ public class JavaTokenizer implements Tokenizer {
         if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
             image = String.valueOf(currentToken.kind);
         }
+        
+        constructorDetector.processToken(currentToken);
+        
         tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
     }
 
@@ -178,4 +187,82 @@ public class JavaTokenizer implements Tokenizer {
             }
         }
     }
+    
+    /**
+     * The {@link ConstructorDetector} consumes token by token and maintains state.
+     * It can detect, whether the current token belongs to a constructor method identifier
+     * and if so, is able to restore it when using ignoreIdentifiers.
+     */
+    private static class ConstructorDetector {
+        private boolean ignoreIdentifiers;
+        
+        private Deque<Integer> classMembersIndentations;
+        private int currentNestingLevel;
+        private boolean constructorCandidate;
+        private String prevIdentifier;
+        
+        public ConstructorDetector(boolean ignoreIdentifiers) {
+            this.ignoreIdentifiers = ignoreIdentifiers;
+            
+            currentNestingLevel = 0;
+            classMembersIndentations = new LinkedList<Integer>();
+        }
+        
+        public void processToken(Token currentToken) {
+            if (!ignoreIdentifiers) {
+                return;
+            }
+            
+            switch (currentToken.kind) {
+            case JavaParserConstants.IDENTIFIER:
+                // Could this be a constructor?
+                if (constructorCandidate && (!classMembersIndentations.isEmpty() && classMembersIndentations.peek().intValue() == currentNestingLevel)) {
+                    prevIdentifier = currentToken.image;
+                }
+                break;
+                
+            case JavaParserConstants.CLASS:
+                // If declaring a class, add a new block nesting level at which constructors may exist
+                classMembersIndentations.push(currentNestingLevel + 1);
+                break;
+                
+            case JavaParserConstants.LBRACE:
+                currentNestingLevel++;
+                break;
+                
+            case JavaParserConstants.RBRACE:
+                // Discard completed blocks
+                if (classMembersIndentations.peek() == currentNestingLevel) {
+                    classMembersIndentations.pop();
+                }
+                currentNestingLevel--;
+                break;
+            }
+            
+            // Can the next token be a constructor identifier?
+            constructorCandidate = currentToken.kind == JavaParserConstants.PRIVATE
+                    || currentToken.kind == JavaParserConstants.PROTECTED
+                    || currentToken.kind == JavaParserConstants.PUBLIC
+                    || currentToken.kind == JavaParserConstants.LBRACE
+                    || currentToken.kind == JavaParserConstants.RBRACE;
+        }
+        
+        public void restoreConstructorToken(Tokens tokenEntries, Token currentToken) {
+            if (!ignoreIdentifiers) {
+                return;
+            }
+            
+            if (prevIdentifier != null) {
+                // was the previous token a constructor? If so, restore the identifier
+                if (currentToken.kind == JavaParserConstants.LPAREN) {
+                    int lastTokenIndex = tokenEntries.size() - 1;
+                    TokenEntry lastToken = tokenEntries.getTokens().get(lastTokenIndex);
+                    tokenEntries.getTokens().set(lastTokenIndex,
+                        new TokenEntry(prevIdentifier, lastToken.getTokenSrcID(), lastToken.getBeginLine()));
+                }
+                
+                prevIdentifier = null;
+            }
+        }
+    }
 }

pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java

@@ -4,7 +4,11 @@
 package net.sourceforge.pmd.cpd;
 
 import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+
 import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;
 
 import org.junit.Test;
 
@@ -203,6 +207,42 @@ public class JavaTokensTokenizerTest {
         assertEquals(1, tokens.size());
     }
 
+    @Test
+    public void testIgnoreIdentifiersDontAffectConstructors() throws Throwable {
+        JavaTokenizer t = new JavaTokenizer();
+        t.setIgnoreAnnotations(false);
+        t.setIgnoreIdentifiers(true);
+        
+        SourceCode sourceCode = new SourceCode(
+            new SourceCode.StringCodeLoader(
+                "package foo.bar.baz;" + PMD.EOL +
+                "public class Foo extends Bar {" + PMD.EOL +
+                	"private Foo notAConstructor;" + PMD.EOL +
+                	"public Foo(int i) { super(i); }" + PMD.EOL +
+                	"private Foo(int i, String s) { super(i, s); }" + PMD.EOL +
+                	"/* default */ Foo(int i, String s, Object o) { super(i, s, o); }" + PMD.EOL +
+                	"private static class Inner {" + PMD.EOL +
+                		"Inner() { System.out.println(\"Guess who?\"); }" + PMD.EOL +
+                	"}" + PMD.EOL +
+                "}" +PMD.EOL
+
+            ));
+        Tokens tokens = new Tokens();
+        t.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        List<TokenEntry> tokenList = tokens.getTokens();
+        
+        // Member variable of type Foo
+        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(7).toString());
+        // Public constructor
+        assertEquals("Foo", tokenList.get(10).toString());
+        // Private constructor
+        assertEquals("Foo", tokenList.get(22).toString());
+        // Package-private constructor
+        assertEquals("Foo", tokenList.get(38).toString());
+        // Inner class constructor
+        assertEquals("Inner", tokenList.get(64).toString());
+    }
 
     public static junit.framework.Test suite() {
         return new junit.framework.JUnit4TestAdapter(JavaTokensTokenizerTest.class);