diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index 60a7aab98d..aad0d2379a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -6,10 +6,9 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Token; -import net.sourceforge.pmd.cpd.token.GenericAntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; @@ -26,16 +25,16 @@ public abstract class AntlrTokenizer implements Tokenizer { AntlrTokenManager tokenManager = getLexerForSource(sourceCode); try { - GenericAntlrToken token = (GenericAntlrToken) tokenManager.getNextToken(); + AntlrToken token = (AntlrToken) tokenManager.getNextToken(); while (token.getType() != Token.EOF) { - if (token.getChannel() != Lexer.HIDDEN) { + if (!token.isHidden()) { final TokenEntry tokenEntry = new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine()); tokenEntries.add(tokenEntry); } - token = (GenericAntlrToken) tokenManager.getNextToken(); + token = (AntlrToken) tokenManager.getNextToken(); } } catch (final AntlrTokenManager.ANTLRSyntaxError err) { // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/GenericAntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java similarity index 82% rename from pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/GenericAntlrToken.java rename to pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java index 85336524c2..7eccf76126 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/GenericAntlrToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.cpd.token; +import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Token; import net.sourceforge.pmd.lang.ast.GenericToken; @@ -13,10 +14,10 @@ import com.beust.jcommander.internal.Nullable; /** * Generic Antlr representation of a token. */ -public class GenericAntlrToken implements GenericToken { +public class AntlrToken implements GenericToken { private final Token token; - private final GenericAntlrToken previousComment; + private final AntlrToken previousComment; /** * Constructor @@ -24,7 +25,7 @@ public class GenericAntlrToken implements GenericToken { * @param token The antlr token implementation * @param previousComment The previous comment */ - public GenericAntlrToken(final Token token, @Nullable final GenericAntlrToken previousComment) { + public AntlrToken(final Token token, @Nullable final AntlrToken previousComment) { this.token = token; this.previousComment = previousComment; } @@ -69,7 +70,7 @@ public class GenericAntlrToken implements GenericToken { return token.getType(); } - public int getChannel() { - return token.getChannel(); + public boolean isHidden() { + return token.getChannel() == Lexer.HIDDEN; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java index 789b0f555f..0bf95b3336 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java @@ -8,12 +8,8 @@ import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; - -import net.sourceforge.pmd.cpd.token.GenericAntlrToken; - -import com.beust.jcommander.internal.Nullable; +import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.TokenManager; /** @@ -22,31 +18,27 @@ import net.sourceforge.pmd.lang.TokenManager; public class AntlrTokenManager implements TokenManager { private final Lexer lexer; private String fileName; - private final String commentToken; - private GenericAntlrToken previousComment; + private AntlrToken previousToken; /** * Constructor * * @param lexer The lexer * @param fileName The file name - * @param commentToken The list of all comment tokens on the grammar */ - public AntlrTokenManager(final Lexer lexer, final String fileName, @Nullable final String commentToken) { + public AntlrTokenManager(final Lexer lexer, final String fileName) { this.lexer = lexer; this.fileName = fileName; - this.commentToken = commentToken; resetListeners(); } @Override public Object getNextToken() { - final Token token = lexer.nextToken(); - if (isCommentToken(token.getText())) { - previousComment = new GenericAntlrToken(token, previousComment); - } + final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null; + final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment); + previousToken = currentToken; - return new GenericAntlrToken(token, previousComment); + return currentToken; } @Override @@ -63,10 +55,6 @@ public class AntlrTokenManager implements TokenManager { lexer.addErrorListener(new ErrorHandler()); } - private boolean isCommentToken(final String text) { - return commentToken != null && text != null && text.startsWith(commentToken); - } - private static class ErrorHandler extends BaseErrorListener { @Override diff --git a/pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java b/pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java index b7063a0d6d..d97b1e6426 100644 --- a/pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java +++ b/pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java @@ -14,11 +14,9 @@ import net.sourceforge.pmd.lang.swift.antlr4.SwiftLexer; */ public class SwiftTokenizer extends AntlrTokenizer { - private static final String COMMENT_TOKEN = "//"; - @Override protected AntlrTokenManager getLexerForSource(final SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); - return new AntlrTokenManager(new SwiftLexer(charStream), sourceCode.getFileName(), COMMENT_TOKEN); + return new AntlrTokenManager(new SwiftLexer(charStream), sourceCode.getFileName()); } }