diff --git a/justfile b/justfile deleted file mode 100644 index bac7611772..0000000000 --- a/justfile +++ /dev/null @@ -1,54 +0,0 @@ - - -pmdJavaDeps := "pmd-core,pmd-lang-test,pmd-test,pmd-java" -commonBuildOpts := "-Dkotlin.compiler.incremental" - -genJavaAst: - rm -f pmd-java/target/generated-sources/javacc/last-generated-timestamp - mvnd generate-sources -pl pmd-java - -reGenAllSources: - rm -rf pmd-*/target/generated-sources - rm pmd-*/target/last-generated-timestamp - mvnd generate-sources - -install MOD: - mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl "pmd-{{MOD}}" - -alias i := install - -cleanInstallEverything *FLAGS: - mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}} - -installEverything *FLAGS: - mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}} - -testCore *FLAGS: - mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}} - -testJava *FLAGS: - mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-java {{FLAGS}} - -installJavaAndDeps *FLAGS: - mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl {{pmdJavaDeps}} {{FLAGS}} - -installJava *FLAGS: - mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-java {{FLAGS}} - - -lintChanged DIFF="master" *FLAGS="": - #!/bin/env zsh - changed=$(git diff --name-only {{DIFF}}) - changed=${(f)changed} - projects=${changed%%/*} # remove all but first segment - echo $projects - # todo filter to pmd-* - mvnd checkstyle:check pmd:check -pl $projects -fae - - -lint projects="pmd-java": - mvnd checkstyle:check pmd:check -pl {{projects}} -fae - -lintAll *FLAGS: - mvnd checkstyle:check pmd:check -fae {{FLAGS}} - diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizerTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizerTest.java index 76467e63b1..4c93a423d4 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizerTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizerTest.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.lang.apex.cpd; +import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.jupiter.api.Test; import net.sourceforge.pmd.cpd.Tokenizer; @@ -44,6 +45,11 @@ class ApexTokenizerTest extends CpdTextComparisonTest { return properties(true); } + @Override + public @NonNull LanguagePropertyConfig defaultProperties() { + return properties(false); + } + private LanguagePropertyConfig properties(boolean caseSensitive) { return properties -> properties.setProperty(Tokenizer.CPD_CASE_SENSITIVE, caseSensitive); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java index 5644159e34..508606b2c2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java @@ -41,16 +41,24 @@ public interface Tokenizer { .build(); PropertyDescriptor CPD_CASE_SENSITIVE = PropertyFactory.booleanProperty("cpdCaseSensitive") - .defaultValue(false) - .desc("Whether CPD should ignore the case of tokens. Affects all tokens.") + .defaultValue(true) + .desc("Whether CPD should respect the case of tokens. Affects all tokens.") .build(); @Deprecated // TODO what to do with this? String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif"; + /** + * Tokenize the source code and record tokens using the provided token factory. + * Implementations should not add an EOF token at the end. + */ void tokenize(TextDocument document, TokenFactory tokens) throws IOException; + /** + * Wraps a call to {@link #tokenize(TextDocument, TokenFactory)} to properly + * create and close the token factory. + */ static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException { try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) { tokenizer.tokenize(textDocument, tf); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java index ebc86025ca..ba3dc0292b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.cpd; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -45,15 +44,11 @@ public class Tokens { return images.entrySet().stream().filter(it -> it.getValue() == i).findFirst().map(Entry::getKey).orElse(null); } - public TokenEntry peekLastToken() { - return get(size() - 1); + TokenEntry peekLastToken() { + return getToken(size() - 1); } - public Iterator iterator() { - return tokens.iterator(); - } - - private TokenEntry get(int index) { + private TokenEntry getToken(int index) { return tokens.get(index); } @@ -61,8 +56,8 @@ public class Tokens { return tokens.size(); } - public TokenEntry getEndToken(TokenEntry mark, Match match) { - return get(mark.getIndex() + match.getTokenCount() - 1); + TokenEntry getEndToken(TokenEntry mark, Match match) { + return getToken(mark.getIndex() + match.getTokenCount() - 1); } public List getTokens() { @@ -70,15 +65,12 @@ public class Tokens { } TokenEntry addToken(String image, String fileName, int startLine, int startCol, int endLine, int endCol) { - TokenEntry newToken = new TokenEntry(getImageId(image), fileName, - startLine, startCol, - endLine, endCol, - tokens.size()); + TokenEntry newToken = new TokenEntry(getImageId(image), fileName, startLine, startCol, endLine, endCol, tokens.size()); add(newToken); return newToken; } - public State savePoint() { + State savePoint() { return new State(this); } @@ -89,15 +81,16 @@ public class Tokens { static final class State { private final int tokenCount; - private final int tokensMapSize; + private final int curImageId; State(Tokens tokens) { this.tokenCount = tokens.tokens.size(); - this.tokensMapSize = tokens.images.size(); + this.curImageId = tokens.curImageId; } public void restore(Tokens tokens) { - tokens.images.entrySet().removeIf(e -> e.getValue() > tokensMapSize); + tokens.images.entrySet().removeIf(e -> e.getValue() >= curImageId); + tokens.curImageId = this.curImageId; final List entries = tokens.getTokens(); entries.subList(tokenCount, entries.size()).clear(); diff --git a/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt b/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt index f73386a117..6458cbcf52 100644 --- a/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt +++ b/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt @@ -94,7 +94,7 @@ abstract class CpdTextComparisonTest( var curLine = -1 - for (token in tokens.iterator()) { + for (token in tokens.tokens) { if (token.isEof) { append("EOF").appendLine() diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfLanguageModule.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfLanguageModule.java index 3504d228e2..cba2e8c13d 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfLanguageModule.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfLanguageModule.java @@ -5,7 +5,7 @@ package net.sourceforge.pmd.lang.vf; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.cpd.VfTokenizer; +import net.sourceforge.pmd.lang.vf.cpd.VfTokenizer; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizer.java similarity index 97% rename from pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java rename to pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizer.java index 72d16b342c..f79bcf48cb 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizer.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.vf.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; diff --git a/pmd-visualforce/src/test/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizerTest.java b/pmd-visualforce/src/test/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizerTest.java index 07842f43c6..0e67555e1e 100644 --- a/pmd-visualforce/src/test/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizerTest.java +++ b/pmd-visualforce/src/test/java/net/sourceforge/pmd/lang/vf/cpd/VfTokenizerTest.java @@ -12,7 +12,7 @@ import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest; class VfTokenizerTest extends CpdTextComparisonTest { VfTokenizerTest() { - super(".page"); + super("vf", ".page"); } @Test