[java] Properly handle enums with ignore identifiers

- Right now it throws an NPE
 - Having fixed the NPE, it threw an ArrayIndexOutOfBounds
 - Finally, it was just not working as expected
 - Fixes https://sourceforge.net/p/pmd/bugs/1542/

pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java

@@ -38,12 +38,7 @@ public class TokenEntry implements Comparable<TokenEntry> {
     }
 
     public TokenEntry(String image, String tokenSrcID, int beginLine) {
-        Integer i = TOKENS.get().get(image);
+        setImage(image);
-        if (i == null) {
-            i = TOKENS.get().size() + 1;
-            TOKENS.get().put(image, i);
-        }
-        this.identifier = i.intValue();
         this.tokenSrcID = tokenSrcID;
         this.beginLine = beginLine;
         this.index = TOKEN_COUNT.get().getAndIncrement();
@@ -128,4 +123,13 @@ public class TokenEntry implements Comparable<TokenEntry> {
         }
         return "--unkown--";
     }
+
+    final void setImage(String image) {
+        Integer i = TOKENS.get().get(image);
+        if (i == null) {
+            i = TOKENS.get().size() + 1;
+            TOKENS.get().put(image, i);
+        }
+        this.identifier = i.intValue();
+    }
 }

pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java

@@ -196,16 +196,16 @@ public class JavaTokenizer implements Tokenizer {
     private static class ConstructorDetector {
         private boolean ignoreIdentifiers;
         
-        private Deque<Integer> classMembersIndentations;
+        private Deque<TypeDeclaration> classMembersIndentations;
         private int currentNestingLevel;
-        private boolean constructorCandidate;
+        private boolean storeNextIdentifier;
         private String prevIdentifier;
         
         public ConstructorDetector(boolean ignoreIdentifiers) {
             this.ignoreIdentifiers = ignoreIdentifiers;
             
             currentNestingLevel = 0;
-            classMembersIndentations = new LinkedList<Integer>();
+            classMembersIndentations = new LinkedList<TypeDeclaration>();
         }
         
         public void processToken(Token currentToken) {
@@ -215,15 +215,21 @@ public class JavaTokenizer implements Tokenizer {
             
             switch (currentToken.kind) {
             case JavaParserConstants.IDENTIFIER:
-                // Could this be a constructor?
+                if ("enum".equals(currentToken.image)) {
-                if (constructorCandidate && (!classMembersIndentations.isEmpty() && classMembersIndentations.peek().intValue() == currentNestingLevel)) {
+                    // If declaring an enum, add a new block nesting level at which constructors may exist
-                    prevIdentifier = currentToken.image;
+                    pushTypeDeclaration();
+                } else if (storeNextIdentifier) {
+                    classMembersIndentations.peek().name = currentToken.image;
+                    storeNextIdentifier = false;
                 }
+                
+                // Store this token
+                prevIdentifier = currentToken.image;
                 break;
                 
             case JavaParserConstants.CLASS:
                 // If declaring a class, add a new block nesting level at which constructors may exist
-                classMembersIndentations.push(currentNestingLevel + 1);
+                pushTypeDeclaration();
                 break;
                 
             case JavaParserConstants.LBRACE:
@@ -232,19 +238,19 @@ public class JavaTokenizer implements Tokenizer {
                 
             case JavaParserConstants.RBRACE:
                 // Discard completed blocks
-                if (classMembersIndentations.peek() == currentNestingLevel) {
+                if (!classMembersIndentations.isEmpty() &&
+                        classMembersIndentations.peek().indentationLevel == currentNestingLevel) {
                     classMembersIndentations.pop();
                 }
                 currentNestingLevel--;
                 break;
             }
+        }
         
-            // Can the next token be a constructor identifier?
+        private void pushTypeDeclaration() {
-            constructorCandidate = currentToken.kind == JavaParserConstants.PRIVATE
+            TypeDeclaration cd = new TypeDeclaration(currentNestingLevel + 1);
-                    || currentToken.kind == JavaParserConstants.PROTECTED
+            classMembersIndentations.push(cd);
-                    || currentToken.kind == JavaParserConstants.PUBLIC
+            storeNextIdentifier = true;
-                    || currentToken.kind == JavaParserConstants.LBRACE
-                    || currentToken.kind == JavaParserConstants.RBRACE;
         }
 
         public void restoreConstructorToken(Tokens tokenEntries, Token currentToken) {
@@ -252,17 +258,24 @@ public class JavaTokenizer implements Tokenizer {
                 return;
             }
             
-            if (prevIdentifier != null) {
-                // was the previous token a constructor? If so, restore the identifier
             if (currentToken.kind == JavaParserConstants.LPAREN) {
+                // was the previous token a constructor? If so, restore the identifier
+                if (!classMembersIndentations.isEmpty()
+                        && classMembersIndentations.peek().name.equals(prevIdentifier)) {
                     int lastTokenIndex = tokenEntries.size() - 1;
                     TokenEntry lastToken = tokenEntries.getTokens().get(lastTokenIndex);
-                    tokenEntries.getTokens().set(lastTokenIndex,
+                    lastToken.setImage(prevIdentifier);
-                        new TokenEntry(prevIdentifier, lastToken.getTokenSrcID(), lastToken.getBeginLine()));
+                }
+            }
+        }
     }
     
-                prevIdentifier = null;
+    private static class TypeDeclaration {
-            }
+        int indentationLevel;
+        String name;
+        
+        public TypeDeclaration(int indentationLevel) {
+            this.indentationLevel = indentationLevel;
         }
     }
 }

pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java

@@ -224,7 +224,7 @@ public class JavaTokensTokenizerTest {
                 	"private static class Inner {" + PMD.EOL +
                 		"Inner() { System.out.println(\"Guess who?\"); }" + PMD.EOL +
                 	"}" + PMD.EOL +
-                "}" +PMD.EOL
+                "}" + PMD.EOL
 
             ));
         Tokens tokens = new Tokens();
@@ -244,6 +244,35 @@ public class JavaTokensTokenizerTest {
         assertEquals("Inner", tokenList.get(64).toString());
     }
     
+    @Test
+    public void testIgnoreIdentifiersHandlesEnums() throws Throwable {
+        JavaTokenizer t = new JavaTokenizer();
+        t.setIgnoreAnnotations(false);
+        t.setIgnoreIdentifiers(true);
+        
+        SourceCode sourceCode = new SourceCode(
+            new SourceCode.StringCodeLoader(
+                "package foo.bar.baz;" + PMD.EOL +
+                "public enum Foo {" + PMD.EOL +
+                	"BAR(1)," + PMD.EOL +
+                	"BAZ(2);" + PMD.EOL +
+                	"Foo(int val) {" + PMD.EOL +
+                	"}" + PMD.EOL +
+                "}" + PMD.EOL
+
+            ));
+        Tokens tokens = new Tokens();
+        t.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        List<TokenEntry> tokenList = tokens.getTokens();
+        
+        // Enum member
+        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(4).toString());
+        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(9).toString());
+        // Enum constructor
+        assertEquals("Foo", tokenList.get(13).toString());
+    }
+
     public static junit.framework.Test suite() {
         return new junit.framework.JUnit4TestAdapter(JavaTokensTokenizerTest.class);
     }