Added AntlrTokenFilter

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java

@@ -4,13 +4,15 @@
 
 package net.sourceforge.pmd.cpd;
 
-import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.CharStreams;
-import org.antlr.v4.runtime.Token;
-
+import net.sourceforge.pmd.cpd.token.AntlrHiddenTokenFilter;
 import net.sourceforge.pmd.cpd.token.AntlrToken;
+import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CharStreams;
+
+import static org.antlr.v4.runtime.Token.EOF;
 
 /**
  * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar.
@@ -23,18 +25,13 @@ public abstract class AntlrTokenizer implements Tokenizer {
     public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) {
 
         AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
+        AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager);
 
         try {
-            AntlrToken token = (AntlrToken) tokenManager.getNextToken();
-
-            while (token.getType() != Token.EOF) {
-                if (!token.isHidden()) {
-                    final TokenEntry tokenEntry =
-                            new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine());
-
-                    tokenEntries.add(tokenEntry);
-                }
-                token = (AntlrToken) tokenManager.getNextToken();
+            AntlrToken currentToken = tokenFilter.getNextToken();
+            while (currentToken != null && currentToken.getType() != EOF) {
+                processToken(tokenEntries, tokenManager.getFileName(), currentToken);
+                currentToken = tokenFilter.getNextToken();
             }
         } catch (final AntlrTokenManager.ANTLRSyntaxError err) {
             // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
@@ -47,8 +44,17 @@ public abstract class AntlrTokenizer implements Tokenizer {
         }
     }
 
+    protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) {
+        return new AntlrHiddenTokenFilter(tokenManager);
+    }
+
     /* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) {
         StringBuilder buffer = sourceCode.getCodeBuffer();
         return CharStreams.fromString(buffer.toString());
     }
+
+    private void processToken(Tokens tokenEntries, String fileName, AntlrToken token) {
+        final TokenEntry tokenEntry = new TokenEntry(token.getImage(), fileName, token.getBeginLine());
+        tokenEntries.add(tokenEntry);
+    }
 }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java

@@ -0,0 +1,16 @@
+package net.sourceforge.pmd.cpd.token;
+
+import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
+
+public class AntlrHiddenTokenFilter extends AntlrTokenFilter {
+
+    public AntlrHiddenTokenFilter(final AntlrTokenManager tokenManager) {
+        super(tokenManager);
+    }
+
+    @Override
+    public AntlrToken getNextToken() {
+        AntlrToken token = super.getNextToken();
+        return token.isHidden() ? getNextToken() : token;
+    }
+}

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java

@@ -0,0 +1,17 @@
+package net.sourceforge.pmd.cpd.token;
+
+import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
+
+public class AntlrTokenFilter implements TokenFilter {
+
+    private TokenFilter tokenFilter;
+
+    public AntlrTokenFilter(final AntlrTokenManager tokenManager) {
+        this.tokenFilter = new JavaCCTokenFilter(tokenManager);
+    }
+
+    @Override
+    public AntlrToken getNextToken() {
+        return (AntlrToken) tokenFilter.getNextToken();
+    }
+}

pmd-go/src/test/java/net/sourceforge/pmd/cpd/GoCPDTokenizerTest.java

@@ -0,0 +1,36 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
+
+public class GoCPDTokenizerTest extends AbstractTokenizerTest {
+
+    private static final String FILENAME = "hello.go";
+
+    @Before
+    @Override
+    public void buildTokenizer() throws IOException {
+        this.tokenizer = new GoTokenizer();
+        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
+    }
+
+    @Override
+    public String getSampleCode() throws IOException {
+        return IOUtils.toString(GoTokenizer.class.getResourceAsStream(FILENAME));
+    }
+
+    @Test
+    public void tokenizeTest() throws IOException {
+        this.expectedTokenCount = 23;
+        super.tokenizeTest();
+    }
+}

pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* CPD-OFF */
+package main
+/* CPD-ON */
+
+import (
+	"fmt"
+
+	"github.com/golang/example/stringutil"
+)
+
+func main() {
+	fmt.Println(stringutil.Reverse("!selpmaxe oG ,olleH"))
+}