Merge branch 'pr-1338'

docs/pages/release_notes.md

@@ -14,6 +14,13 @@ This is a {{ site.pmd.release_type }} release.
 
 ### New and noteworthy
 
+#### Improved Golang CPD Support
+
+Thanks to the work of [ITBA](https://www.itba.edu.ar/) students [Matías Fraga](https://github.com/matifraga),
+[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini),
+Golang is now backed by a proper Antlr Grammar. This means CPD is now better at detecting duplicates,
+as comments are recognized as such and ignored.
+
 ### Fixed Issues
 
 *   java
@@ -28,6 +35,7 @@ This is a {{ site.pmd.release_type }} release.
 
 ### External Contributions
 
+*   [#1338](https://github.com/pmd/pmd/pull/1338): \[core] [cpd] Generalize ANTLR tokens preparing support for ANTLR token filter - [Matías Fraga](https://github.com/matifraga) and [Tomi De Lucca](https://github.com/tomidelucca)
 *   [#1366](https://github.com/pmd/pmd/pull/1366): \[java] Static Modifier on Internal Interface pmd #1356 - [avishvat](https://github.com/vishva007)
 *   [#1368](https://github.com/pmd/pmd/pull/1368): \[doc] Updated outdated note in the building documentation. - [Maikel Steneker](https://github.com/maikelsteneker)
 *   [#1374](https://github.com/pmd/pmd/pull/1374): \[java] Simplify check for 'Test' annotation in JUnitTestsShouldIncludeAssertRule. - [Will Winder](https://github.com/winder)

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java

@@ -6,9 +6,9 @@ package net.sourceforge.pmd.cpd;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
-import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.Token;
 
+import net.sourceforge.pmd.cpd.token.AntlrToken;
 import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
@@ -25,16 +25,16 @@ public abstract class AntlrTokenizer implements Tokenizer {
         AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
 
         try {
-            Token token = (Token) tokenManager.getNextToken();
+            AntlrToken token = (AntlrToken) tokenManager.getNextToken();
 
             while (token.getType() != Token.EOF) {
-                if (token.getChannel() != Lexer.HIDDEN) {
+                if (!token.isHidden()) {
                     final TokenEntry tokenEntry =
-                            new TokenEntry(token.getText(), tokenManager.getFileName(), token.getLine());
+                            new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine());
 
                     tokenEntries.add(tokenEntry);
                 }
-                token = (Token) tokenManager.getNextToken();
+                token = (AntlrToken) tokenManager.getNextToken();
             }
         } catch (final AntlrTokenManager.ANTLRSyntaxError err) {
             // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java

@@ -0,0 +1,76 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd.token;
+
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.Token;
+
+import net.sourceforge.pmd.lang.ast.GenericToken;
+
+import com.beust.jcommander.internal.Nullable;
+
+/**
+ * Generic Antlr representation of a token.
+ */
+public class AntlrToken implements GenericToken {
+
+    private final Token token;
+    private final AntlrToken previousComment;
+
+    /**
+     * Constructor
+     *
+     * @param token The antlr token implementation
+     * @param previousComment The previous comment
+     */
+    public AntlrToken(final Token token, @Nullable final AntlrToken previousComment) {
+        this.token = token;
+        this.previousComment = previousComment;
+    }
+
+    @Override
+    public GenericToken getNext() {
+        // Antlr implementation does not require this
+        return null;
+    }
+
+    @Override
+    public GenericToken getPreviousComment() {
+        return previousComment;
+    }
+
+    @Override
+    public String getImage() {
+        return token.getText();
+    }
+
+    @Override
+    public int getBeginLine() {
+        return token.getLine();
+    }
+
+    @Override
+    public int getEndLine() {
+        return token.getLine();
+    }
+
+    @Override
+    public int getBeginColumn() {
+        return token.getCharPositionInLine();
+    }
+
+    @Override
+    public int getEndColumn() {
+        return token.getCharPositionInLine() + token.getStopIndex() - token.getStartIndex();
+    }
+
+    public int getType() {
+        return token.getType();
+    }
+
+    public boolean isHidden() {
+        return token.getChannel() == Lexer.HIDDEN;
+    }
+}

pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java

@@ -9,6 +9,7 @@ import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 
+import net.sourceforge.pmd.cpd.token.AntlrToken;
 import net.sourceforge.pmd.lang.TokenManager;
 
 /**
@@ -17,6 +18,7 @@ import net.sourceforge.pmd.lang.TokenManager;
 public class AntlrTokenManager implements TokenManager {
     private final Lexer lexer;
     private String fileName;
+    private AntlrToken previousToken;
 
     /**
      * Constructor
@@ -32,7 +34,11 @@ public class AntlrTokenManager implements TokenManager {
 
     @Override
     public Object getNextToken() {
-        return lexer.nextToken();
+        final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null;
+        final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment);
+        previousToken = currentToken;
+
+        return currentToken;
     }
 
     @Override
@@ -49,7 +55,6 @@ public class AntlrTokenManager implements TokenManager {
         lexer.addErrorListener(new ErrorHandler());
     }
 
-
     private static class ErrorHandler extends BaseErrorListener {
 
         @Override