diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 786a3eb383..cd99b5f9a9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -80,6 +80,7 @@ More information is available in [the user documentation](pmd_userdocs_cpd.html# * [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) * [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17) * [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca) +* [#1655](https://github.com/pmd/pmd/pull/1655): \[kotlin] Kotlin tokenizer refactor - [Lucas Soncini](https://github.com/lsoncini) {% endtocmaker %} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java index 7372f15906..c78e191d08 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java @@ -316,7 +316,7 @@ public class PMD { // Make sure the cache is listening for analysis results ctx.getReport().addListener(configuration.getAnalysisCache()); - final RuleSetFactory silentFactoy = new RuleSetFactory(ruleSetFactory, false); + final RuleSetFactory silentFactory = new RuleSetFactory(ruleSetFactory, false); /* * Check if multithreaded support is available. ExecutorService can also @@ -324,9 +324,9 @@ public class PMD { * "-threads 0" command line option. */ if (configuration.getThreads() > 0) { - new MultiThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MultiThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } else { - new MonoThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MonoThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } // Persist the analysis cache diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 0f43a305fb..747619b5f3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -69,7 +69,7 @@ public abstract class BaseTokenFilter implements TokenFi * @param currentToken The token to be analyzed * @see #isLanguageSpecificDiscarding() */ - protected void analyzeToken(final GenericToken currentToken) { + protected void analyzeToken(final T currentToken) { // noop } diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index 17da5ec802..dcfe0da1c0 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -5,10 +5,9 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; @@ -17,46 +16,55 @@ import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; */ public class KotlinTokenizer extends AntlrTokenizer { - private boolean discardingPackageAndImport = false; - @Override protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); - final Lexer lexer = new Kotlin(charStream); - final AntlrTokenManager tokenManager = new AntlrTokenManager(lexer, sourceCode.getFileName()) { - @Override - public Object getNextToken() { - AntlrToken nextToken; - boolean done = false; - do { - nextToken = (AntlrToken) super.getNextToken(); - analyzeTokenStart(nextToken); - if (!nextToken.isHidden() && nextToken.getType() != Kotlin.NL && !isDiscarding()) { - done = true; - } - analyzeTokenEnd(nextToken); - } while (!done && nextToken.getType() != Token.EOF); - return nextToken; - } - }; - return tokenManager; + return new AntlrTokenManager(new Kotlin(charStream), sourceCode.getFileName()); } - private boolean isDiscarding() { - return discardingPackageAndImport; + @Override + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { + return new KotlinTokenFilter(tokenManager); } - private void analyzeTokenStart(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { - discardingPackageAndImport = true; + /** + * The {@link KotlinTokenFilter} extends the {@link AntlrTokenFilter} to discard + * Kotlin-specific tokens. + *

+ * By default, it discards package and import statements, and + * enables annotation-based CPD suppression. + *

+ */ + private static class KotlinTokenFilter extends AntlrTokenFilter { + private boolean discardingPackageAndImport = false; + private boolean discardingNL = false; + + /* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) { + super(tokenManager); } - } - private void analyzeTokenEnd(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { - discardingPackageAndImport = false; + @Override + protected void analyzeToken(final AntlrToken currentToken) { + skipPackageAndImport(currentToken); + skipNewLines(currentToken); + } + + private void skipPackageAndImport(final AntlrToken currentToken) { + final int type = currentToken.getType(); + if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { + discardingPackageAndImport = true; + } else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { + discardingPackageAndImport = false; + } + } + + private void skipNewLines(final AntlrToken currentToken) { + discardingNL = currentToken.getType() == Kotlin.NL; + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return discardingPackageAndImport || discardingNL; } } }