From 9b20ec524af849b06fb8b30bfdd90f7e2198b743 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 27 Jun 2024 21:15:10 +0200 Subject: [PATCH] [core] Remove AntlrLexerBehavior --- .../pmd/cpd/impl/AntlrCpdLexer.java | 11 +------ .../ast/impl/antlr4/AntlrLexerBehavior.java | 32 ------------------- .../pmd/lang/ast/impl/antlr4/AntlrToken.java | 21 +++++------- .../ast/impl/antlr4/AntlrTokenManager.java | 10 +----- .../pmd/lang/tsql/cpd/TSqlCpdLexer.java | 22 +++++-------- 5 files changed, 18 insertions(+), 78 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrLexerBehavior.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrCpdLexer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrCpdLexer.java index a451e8400d..3c06996210 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrCpdLexer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrCpdLexer.java @@ -12,7 +12,6 @@ import org.antlr.v4.runtime.Lexer; import net.sourceforge.pmd.cpd.CpdLexer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrLexerBehavior; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.lang.document.TextDocument; @@ -24,15 +23,7 @@ public abstract class AntlrCpdLexer extends CpdLexerBase { @Override protected final TokenManager makeLexerImpl(TextDocument doc) throws IOException { CharStream charStream = CharStreams.fromReader(doc.newReader(), doc.getFileId().getAbsolutePath()); - return new AntlrTokenManager(getLexerForSource(charStream), doc, getLexerBehavior()); - } - - /** - * Override this method to customize some aspects of the - * lexer. - */ - protected AntlrLexerBehavior getLexerBehavior() { - return new AntlrLexerBehavior(); + return new AntlrTokenManager(getLexerForSource(charStream), doc); } protected abstract Lexer getLexerForSource(CharStream charStream); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrLexerBehavior.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrLexerBehavior.java deleted file mode 100644 index 7d2e8cf24e..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrLexerBehavior.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.antlr4; - -import org.antlr.v4.runtime.Token; - -import net.sourceforge.pmd.cpd.CpdLanguageProperties; - -/** - * Strategy to customize some aspects of the mapping - * from Antlr tokens to PMD/CPD tokens. - */ -public class AntlrLexerBehavior { - - - /** - * Return the image that the token should have, possibly applying a transformation. - * The default just returns {@link Token#getText()}. - * Transformations here are usually normalizations, for instance, mapping - * the image of all keywords to uppercase/lowercase to implement case-insensitivity, - * or replacing the image of literals by a placeholder to implement {@link CpdLanguageProperties#CPD_ANONYMIZE_LITERALS}. - * - * @param token A token from the Antlr Lexer - * - * @return The image - */ - protected String getTokenImage(Token token) { - return token.getText(); - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java index 22a4fcbdc3..4a12e8388d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java @@ -33,23 +33,18 @@ public class AntlrToken implements GenericToken { * @param token The antlr token implementation * @param previousComment The previous comment * @param textDoc The text document - */ - AntlrToken(final Token token, final AntlrToken previousComment, TextDocument textDoc, AntlrLexerBehavior behavior) { - this.previousComment = previousComment; - this.textDoc = textDoc; - this.image = behavior.getTokenImage(token); - this.startOffset = token.getStartIndex(); - this.endOffset = token.getStopIndex() + 1; // exclusive - this.channel = token.getChannel(); - this.kind = token.getType(); - } - - /** + * * @deprecated Don't create antlr tokens directly, use an {@link AntlrTokenManager} */ @Deprecated public AntlrToken(final Token token, final AntlrToken previousComment, TextDocument textDoc) { - this(token, previousComment, textDoc, new AntlrLexerBehavior()); + this.previousComment = previousComment; + this.textDoc = textDoc; + this.image = token.getText(); + this.startOffset = token.getStartIndex(); + this.endOffset = token.getStopIndex() + 1; // exclusive + this.channel = token.getChannel(); + this.kind = token.getType(); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java index 6c54e3ef05..5828195850 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java @@ -20,20 +20,12 @@ public class AntlrTokenManager implements TokenManager { private final Lexer lexer; private final TextDocument textDoc; - private final AntlrLexerBehavior behavior; private AntlrToken previousToken; public AntlrTokenManager(final Lexer lexer, final TextDocument textDocument) { - this(lexer, textDocument, new AntlrLexerBehavior()); - } - - public AntlrTokenManager(final Lexer lexer, - final TextDocument textDocument, - final AntlrLexerBehavior behavior) { this.lexer = lexer; this.textDoc = textDocument; - this.behavior = behavior; resetListeners(); } @@ -48,7 +40,7 @@ public class AntlrTokenManager implements TokenManager { private AntlrToken getNextTokenFromAnyChannel() { final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null; - final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment, textDoc, this.behavior); + final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment, textDoc); if (previousToken != null) { previousToken.next = currentToken; } diff --git a/pmd-tsql/src/main/java/net/sourceforge/pmd/lang/tsql/cpd/TSqlCpdLexer.java b/pmd-tsql/src/main/java/net/sourceforge/pmd/lang/tsql/cpd/TSqlCpdLexer.java index deea25c4fb..3e82239cda 100644 --- a/pmd-tsql/src/main/java/net/sourceforge/pmd/lang/tsql/cpd/TSqlCpdLexer.java +++ b/pmd-tsql/src/main/java/net/sourceforge/pmd/lang/tsql/cpd/TSqlCpdLexer.java @@ -8,10 +8,9 @@ import java.util.Locale; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; import net.sourceforge.pmd.cpd.impl.AntlrCpdLexer; -import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrLexerBehavior; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.tsql.ast.TSqlLexer; /** @@ -25,17 +24,12 @@ public class TSqlCpdLexer extends AntlrCpdLexer { } @Override - protected AntlrLexerBehavior getLexerBehavior() { - return new AntlrLexerBehavior() { - @Override - protected String getTokenImage(Token token) { - if (token.getType() == TSqlLexer.STRING) { - // This path is for case-sensitive tokens - return super.getTokenImage(token); - } - // normalize case sensitive tokens - return token.getText().toUpperCase(Locale.ROOT); - } - }; + protected String getImage(AntlrToken token) { + if (token.getKind() == TSqlLexer.STRING) { + // This path is for case-sensitive tokens + return token.getImage(); + } + // normalize case-insensitive tokens + return token.getImage().toUpperCase(Locale.ROOT); } }