From 72f2e425b278ea7cf008b664aa54134071e8338d Mon Sep 17 00:00:00 2001 From: lsoncini Date: Sat, 16 Feb 2019 15:49:03 -0300 Subject: [PATCH 1/4] KotlinTokenizer refactor --- .../main/java/net/sourceforge/pmd/PMD.java | 6 +- .../sourceforge/pmd/cpd/KotlinTokenizer.java | 75 +++++++++++-------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java index 7372f15906..c78e191d08 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java @@ -316,7 +316,7 @@ public class PMD { // Make sure the cache is listening for analysis results ctx.getReport().addListener(configuration.getAnalysisCache()); - final RuleSetFactory silentFactoy = new RuleSetFactory(ruleSetFactory, false); + final RuleSetFactory silentFactory = new RuleSetFactory(ruleSetFactory, false); /* * Check if multithreaded support is available. ExecutorService can also @@ -324,9 +324,9 @@ public class PMD { * "-threads 0" command line option. */ if (configuration.getThreads() > 0) { - new MultiThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MultiThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } else { - new MonoThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MonoThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } // Persist the analysis cache diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index 17da5ec802..b4ee895e5f 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -5,10 +5,9 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; @@ -17,46 +16,56 @@ import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; */ public class KotlinTokenizer extends AntlrTokenizer { - private boolean discardingPackageAndImport = false; - @Override protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); - final Lexer lexer = new Kotlin(charStream); - final AntlrTokenManager tokenManager = new AntlrTokenManager(lexer, sourceCode.getFileName()) { - @Override - public Object getNextToken() { - AntlrToken nextToken; - boolean done = false; - do { - nextToken = (AntlrToken) super.getNextToken(); - analyzeTokenStart(nextToken); - if (!nextToken.isHidden() && nextToken.getType() != Kotlin.NL && !isDiscarding()) { - done = true; - } - analyzeTokenEnd(nextToken); - } while (!done && nextToken.getType() != Token.EOF); - return nextToken; - } - }; - return tokenManager; + return new AntlrTokenManager(new Kotlin(charStream), sourceCode.getFileName()); } - private boolean isDiscarding() { - return discardingPackageAndImport; + @Override + protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + return new KotlinTokenFilter(tokenManager); } - private void analyzeTokenStart(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { - discardingPackageAndImport = true; + /** + * The {@link KotlinTokenFilter} extends the {@link AntlrTokenFilter} to discard + * Kotlin-specific tokens. + *

+ * By default, it discards package and import statements, and + * enables annotation-based CPD suppression. + *

+ */ + private static class KotlinTokenFilter extends AntlrTokenFilter { + private boolean discardingPackageAndImport = false; + private boolean discardingNL = false; + + /* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) { + super(tokenManager); } - } - private void analyzeTokenEnd(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { - discardingPackageAndImport = false; + @Override + protected void analyzeToken(final AntlrToken currentToken) { + super.analyzeToken(currentToken); + skipPackageAndImport(currentToken); + skipNewLines(currentToken); + } + + private void skipPackageAndImport(final AntlrToken currentToken) { + final int type = currentToken.getType(); + if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { + discardingPackageAndImport = true; + } else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { + discardingPackageAndImport = false; + } + } + + private void skipNewLines(final AntlrToken currentToken) { + discardingNL = currentToken.getType() == Kotlin.NL; + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return discardingPackageAndImport || discardingNL || super.isLanguageSpecificDiscarding(); } } } From 22aaa70983c093d6ca5e3f1e55a57903716e1424 Mon Sep 17 00:00:00 2001 From: Matias Fraga Date: Sat, 16 Feb 2019 16:20:03 -0300 Subject: [PATCH 2/4] Add final modifier --- .../src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index b4ee895e5f..af87b2f783 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -23,7 +23,7 @@ public class KotlinTokenizer extends AntlrTokenizer { } @Override - protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { return new KotlinTokenFilter(tokenManager); } From 55c2c00e7aceb4f4794d3e148fb1580c0e982e58 Mon Sep 17 00:00:00 2001 From: lsoncini Date: Sun, 17 Feb 2019 20:28:57 -0300 Subject: [PATCH 3/4] merge fixes --- .../sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java | 2 +- .../src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 0f43a305fb..747619b5f3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -69,7 +69,7 @@ public abstract class BaseTokenFilter implements TokenFi * @param currentToken The token to be analyzed * @see #isLanguageSpecificDiscarding() */ - protected void analyzeToken(final GenericToken currentToken) { + protected void analyzeToken(final T currentToken) { // noop } diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index b4ee895e5f..5ce23f13c4 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -45,7 +45,6 @@ public class KotlinTokenizer extends AntlrTokenizer { @Override protected void analyzeToken(final AntlrToken currentToken) { - super.analyzeToken(currentToken); skipPackageAndImport(currentToken); skipNewLines(currentToken); } @@ -65,7 +64,7 @@ public class KotlinTokenizer extends AntlrTokenizer { @Override protected boolean isLanguageSpecificDiscarding() { - return discardingPackageAndImport || discardingNL || super.isLanguageSpecificDiscarding(); + return discardingPackageAndImport || discardingNL; } } } From 52c4d6ed374f2bc5043e94087526ae0a9780cb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 23:34:15 -0300 Subject: [PATCH 4/4] Update changelog, refs #1655 --- docs/pages/release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 786a3eb383..cd99b5f9a9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -80,6 +80,7 @@ More information is available in [the user documentation](pmd_userdocs_cpd.html# * [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) * [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17) * [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca) +* [#1655](https://github.com/pmd/pmd/pull/1655): \[kotlin] Kotlin tokenizer refactor - [Lucas Soncini](https://github.com/lsoncini) {% endtocmaker %}