diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b99f8d5969..02d57d8465 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,13 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### Improved Golang CPD Support + +Thanks to the work of [ITBA](https://www.itba.edu.ar/) students [Matías Fraga](https://github.com/matifraga), +[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini), +Golang is now backed by a proper Antlr Grammar. This means CPD is now better at detecting duplicates, +as comments are recognized as such and ignored. + ### Fixed Issues * java @@ -28,6 +35,7 @@ This is a {{ site.pmd.release_type }} release. ### External Contributions +* [#1338](https://github.com/pmd/pmd/pull/1338): \[core] [cpd] Generalize ANTLR tokens preparing support for ANTLR token filter - [Matías Fraga](https://github.com/matifraga) and [Tomi De Lucca](https://github.com/tomidelucca) * [#1366](https://github.com/pmd/pmd/pull/1366): \[java] Static Modifier on Internal Interface pmd #1356 - [avishvat](https://github.com/vishva007) * [#1368](https://github.com/pmd/pmd/pull/1368): \[doc] Updated outdated note in the building documentation. - [Maikel Steneker](https://github.com/maikelsteneker) * [#1374](https://github.com/pmd/pmd/pull/1374): \[java] Simplify check for 'Test' annotation in JUnitTestsShouldIncludeAssertRule. - [Will Winder](https://github.com/winder) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index a5082f5037..aad0d2379a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -6,9 +6,9 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Token; +import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; @@ -25,16 +25,16 @@ public abstract class AntlrTokenizer implements Tokenizer { AntlrTokenManager tokenManager = getLexerForSource(sourceCode); try { - Token token = (Token) tokenManager.getNextToken(); + AntlrToken token = (AntlrToken) tokenManager.getNextToken(); while (token.getType() != Token.EOF) { - if (token.getChannel() != Lexer.HIDDEN) { + if (!token.isHidden()) { final TokenEntry tokenEntry = - new TokenEntry(token.getText(), tokenManager.getFileName(), token.getLine()); + new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine()); tokenEntries.add(tokenEntry); } - token = (Token) tokenManager.getNextToken(); + token = (AntlrToken) tokenManager.getNextToken(); } } catch (final AntlrTokenManager.ANTLRSyntaxError err) { // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java new file mode 100644 index 0000000000..7eccf76126 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java @@ -0,0 +1,76 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd.token; + +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Token; + +import net.sourceforge.pmd.lang.ast.GenericToken; + +import com.beust.jcommander.internal.Nullable; + +/** + * Generic Antlr representation of a token. + */ +public class AntlrToken implements GenericToken { + + private final Token token; + private final AntlrToken previousComment; + + /** + * Constructor + * + * @param token The antlr token implementation + * @param previousComment The previous comment + */ + public AntlrToken(final Token token, @Nullable final AntlrToken previousComment) { + this.token = token; + this.previousComment = previousComment; + } + + @Override + public GenericToken getNext() { + // Antlr implementation does not require this + return null; + } + + @Override + public GenericToken getPreviousComment() { + return previousComment; + } + + @Override + public String getImage() { + return token.getText(); + } + + @Override + public int getBeginLine() { + return token.getLine(); + } + + @Override + public int getEndLine() { + return token.getLine(); + } + + @Override + public int getBeginColumn() { + return token.getCharPositionInLine(); + } + + @Override + public int getEndColumn() { + return token.getCharPositionInLine() + token.getStopIndex() - token.getStartIndex(); + } + + public int getType() { + return token.getType(); + } + + public boolean isHidden() { + return token.getChannel() == Lexer.HIDDEN; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java index 8b0e4b0712..0bf95b3336 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java @@ -9,6 +9,7 @@ import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; +import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.TokenManager; /** @@ -17,6 +18,7 @@ import net.sourceforge.pmd.lang.TokenManager; public class AntlrTokenManager implements TokenManager { private final Lexer lexer; private String fileName; + private AntlrToken previousToken; /** * Constructor @@ -32,7 +34,11 @@ public class AntlrTokenManager implements TokenManager { @Override public Object getNextToken() { - return lexer.nextToken(); + final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null; + final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment); + previousToken = currentToken; + + return currentToken; } @Override @@ -49,7 +55,6 @@ public class AntlrTokenManager implements TokenManager { lexer.addErrorListener(new ErrorHandler()); } - private static class ErrorHandler extends BaseErrorListener { @Override