diff --git a/justfile b/justfile index 367f1a64d0..bac7611772 100644 --- a/justfile +++ b/justfile @@ -20,6 +20,9 @@ alias i := install cleanInstallEverything *FLAGS: mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}} +installEverything *FLAGS: + mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}} + testCore *FLAGS: mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}} @@ -46,6 +49,6 @@ lintChanged DIFF="master" *FLAGS="": lint projects="pmd-java": mvnd checkstyle:check pmd:check -pl {{projects}} -fae -lintAll: - mvnd checkstyle:check pmd:check -fae +lintAll *FLAGS: + mvnd checkstyle:check pmd:check -fae {{FLAGS}} diff --git a/pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml b/pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml index 5a76ac24da..3f84018714 100644 --- a/pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml +++ b/pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml @@ -8,7 +8,7 @@ - + diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java index 874da4dd4c..5ef1260d54 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java @@ -4,13 +4,13 @@ package net.sourceforge.pmd.lang.apex; -import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguageModuleBase; import net.sourceforge.pmd.lang.LanguageProcessor; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer; import apex.jorje.services.Version; diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java index cea287a50e..4b175ea6a0 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java @@ -4,32 +4,60 @@ package net.sourceforge.pmd.lang.apex.cpd; +import java.io.IOException; import java.util.Locale; -import org.antlr.v4.runtime.CharStream; +import org.antlr.runtime.ANTLRReaderStream; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.Lexer; +import org.antlr.runtime.Token; +import net.sourceforge.pmd.cpd.TokenFactory; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; +import net.sourceforge.pmd.lang.apex.ApexJorjeLogging; import net.sourceforge.pmd.lang.apex.ApexLanguageProperties; -import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; +import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.document.TextDocument; + +import apex.jorje.parser.impl.ApexLexer; + +public class ApexTokenizer implements Tokenizer { -public class ApexTokenizer extends AntlrTokenizer { private final boolean caseSensitive; public ApexTokenizer(ApexLanguageProperties properties) { this.caseSensitive = properties.getProperty(Tokenizer.CPD_CASE_SENSITIVE); + ApexJorjeLogging.disableLogging(); } @Override - protected String getImage(AntlrToken token) { - if (caseSensitive) { - return token.getImage(); + public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException { + + ANTLRStringStream ass = new ANTLRReaderStream(document.newReader()); + ApexLexer lexer = new ApexLexer(ass) { + @Override + public void emitErrorMessage(String msg) { + throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null); + } + }; + + Token token = lexer.nextToken(); + + while (token.getType() != Token.EOF) { + if (token.getChannel() != Lexer.HIDDEN) { + String tokenText = token.getText(); + if (!caseSensitive) { + tokenText = tokenText.toLowerCase(Locale.ROOT); + } + tokenEntries.recordToken( + tokenText, + token.getLine(), + token.getCharPositionInLine() + 1, + token.getLine(), + token.getCharPositionInLine() + tokenText.length() + 1 + ); + } + token = lexer.nextToken(); } - return token.getImage().toLowerCase(Locale.ROOT); - } - - @Override - protected org.antlr.v4.runtime.Lexer getLexerForSource(CharStream charStream) { - return new com.nawforce.runtime.parsers.ApexLexer(charStream); } } diff --git a/pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java b/pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java index d4d91786bc..5f3ae64ac7 100644 --- a/pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java +++ b/pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java @@ -124,7 +124,7 @@ public class CpdCommand extends AbstractAnalysisPmdSubcommand { final CPDConfiguration configuration = toConfiguration(); - try (CpdAnalysis cpd = new CpdAnalysis(configuration)){ + try (CpdAnalysis cpd = new CpdAnalysis(configuration)) { MutableBoolean hasViolations = new MutableBoolean(); cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty())); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java index 49000eaadd..0d3de7a45d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java @@ -62,8 +62,8 @@ public class AnyTokenizer implements Tokenizer { } @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokens) { - Chars text = sourceCode.getText(); + public void tokenize(TextDocument document, TokenFactory tokens) { + Chars text = document.getText(); Matcher matcher = pattern.matcher(text); int lineNo = 1; int lastLineStart = 0; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java index 5b6ea12628..ba52e4d7c8 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java @@ -62,11 +62,11 @@ import javax.swing.table.TableModel; import net.sourceforge.pmd.PMDVersion; import net.sourceforge.pmd.cpd.renderer.CPDReportRenderer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguageModuleBase.LanguageMetadata; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; public class GUI implements CPDListener { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java index 63645db659..5644159e34 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java @@ -26,10 +26,8 @@ public interface Tokenizer { PropertyDescriptor CPD_ANONYMIZE_IDENTIFIERS = PropertyFactory.booleanProperty("cpdAnonymizeIdentifiers") .defaultValue(false) - .desc("Anonymize identifiers. They are still part of the token stream but all literals appear to have the same value.") + .desc("Anonymize identifiers. They are still part of the token stream but all identifiers appear to have the same value.") .build(); - - PropertyDescriptor CPD_IGNORE_IMPORTS = PropertyFactory.booleanProperty("cpdIgnoreImports") .defaultValue(true) @@ -41,48 +39,17 @@ public interface Tokenizer { .defaultValue(false) .desc("Ignore metadata such as Java annotations or C# attributes.") .build(); - - PropertyDescriptor CPD_CASE_SENSITIVE = PropertyFactory.booleanProperty("cpdCaseSensitive") - .defaultValue(true) + .defaultValue(false) .desc("Whether CPD should ignore the case of tokens. Affects all tokens.") .build(); - String IGNORE_LITERALS = "ignore_literals"; - String IGNORE_IDENTIFIERS = "ignore_identifiers"; - String IGNORE_ANNOTATIONS = "ignore_annotations"; - - /** - * Ignore sequences of literals (e.g, 0,0,0,0...). - */ - String OPTION_IGNORE_LITERAL_SEQUENCES = "net.sourceforge.pmd.cpd.Tokenizer.skipLiteralSequences"; - /** - * Ignore using directives in C#. The default value is false. - */ - String IGNORE_USINGS = "ignore_usings"; - - /** - * Enables or disabled skipping of blocks like a pre-processor. It is a - * boolean property. The default value is true. - * - * @see #OPTION_SKIP_BLOCKS_PATTERN - */ - String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks"; - /** - * Configures the pattern, to find the blocks to skip. It is a string - * property and contains of two parts, separated by {@code |}. The first - * part is the start pattern, the second part is the ending pattern. Default - * value is "{@code #if 0|#endif}". - * - * @see #DEFAULT_SKIP_BLOCKS_PATTERN - */ - String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern"; - + @Deprecated // TODO what to do with this? String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif"; - void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException; + void tokenize(TextDocument document, TokenFactory tokens) throws IOException; static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException { try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/internal/LanguageServiceBase.java b/pmd-core/src/main/java/net/sourceforge/pmd/internal/LanguageServiceBase.java deleted file mode 100644 index 096cbbe603..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/internal/LanguageServiceBase.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.internal; - -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; -import java.util.Set; -import java.util.TreeSet; - -import net.sourceforge.pmd.annotation.InternalApi; - -@InternalApi -public abstract class LanguageServiceBase { - - protected interface NameExtractor { - String getName(T language); - } - - protected final Set languages; - protected final Map languagesByName; - protected final Map languagesByTerseName; - - protected LanguageServiceBase(final Class serviceType, final Comparator comparator, - final NameExtractor nameExtractor, final NameExtractor terseNameExtractor) { - Set sortedLangs = new TreeSet<>(comparator); - // Use current class' classloader instead of the threads context classloader, see https://github.com/pmd/pmd/issues/1788 - ServiceLoader languageLoader = ServiceLoader.load(serviceType, getClass().getClassLoader()); - Iterator iterator = languageLoader.iterator(); - - while (true) { - // this loop is weird, but both hasNext and next may throw ServiceConfigurationError, - // it's more robust that way - try { - if (iterator.hasNext()) { - T language = iterator.next(); - sortedLangs.add(language); - } else { - break; - } - } catch (UnsupportedClassVersionError | ServiceConfigurationError e) { - // Some languages require java8 and are therefore only available - // if java8 or later is used as runtime. - System.err.println("Ignoring language for PMD: " + e.toString()); - } - } - - // using a linked hash map to maintain insertion order - languages = Collections.unmodifiableSet(new LinkedHashSet<>(sortedLangs)); - - // TODO there may be languages with duplicate names - Map byName = new LinkedHashMap<>(); - Map byTerseName = new LinkedHashMap<>(); - for (T language : sortedLangs) { - byName.put(nameExtractor.getName(language), language); - byTerseName.put(terseNameExtractor.getName(language), language); - } - languagesByName = Collections.unmodifiableMap(byName); - languagesByTerseName = Collections.unmodifiableMap(byTerseName); - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/CpdOnlyLanguageModuleBase.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/impl/CpdOnlyLanguageModuleBase.java similarity index 84% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/CpdOnlyLanguageModuleBase.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/impl/CpdOnlyLanguageModuleBase.java index 1db5c8689f..f67f7e7dec 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/CpdOnlyLanguageModuleBase.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/impl/CpdOnlyLanguageModuleBase.java @@ -2,9 +2,11 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang; +package net.sourceforge.pmd.lang.impl; import net.sourceforge.pmd.cpd.Tokenizer; +import net.sourceforge.pmd.lang.LanguageModuleBase; +import net.sourceforge.pmd.lang.LanguagePropertyBundle; /** * Base class for language modules that only support CPD and not PMD. diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java similarity index 58% rename from pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java index b8af18b063..03f4061255 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java @@ -1,4 +1,4 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ @@ -13,12 +13,10 @@ import java.util.Iterator; import org.junit.jupiter.api.Test; +import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.Language; -import net.sourceforge.pmd.lang.LanguagePropertyBundle; -import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.document.TextFile; -import net.sourceforge.pmd.lang.java.JavaLanguageModule; class MatchAlgorithmTest { @@ -38,15 +36,15 @@ class MatchAlgorithmTest { @Test void testSimple() throws IOException { - Language java = JavaLanguageModule.getInstance(); - Tokenizer tokenizer = java.createCpdTokenizer(java.newPropertyBundle()); - String fileName = "Foo.java"; - TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, java.getDefaultVersion()); + Language dummy = DummyLanguageModule.getInstance(); + Tokenizer tokenizer = dummy.createCpdTokenizer(dummy.newPropertyBundle()); + String fileName = "Foo.dummy"; + TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, dummy.getDefaultVersion()); SourceManager sourceManager = new SourceManager(listOf(textFile)); Tokens tokens = new Tokens(); TextDocument sourceCode = sourceManager.get(textFile); Tokenizer.tokenize(tokenizer, sourceCode, tokens); - assertEquals(41, tokens.size()); + assertEquals(44, tokens.size()); MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5); matchAlgorithm.findMatches(); @@ -61,34 +59,10 @@ class MatchAlgorithmTest { assertEquals(3, mark1.getBeginLine()); assertEquals(fileName, mark1.getFilename()); - assertEquals(Chars.wrap(LINE_3), sourceManager.getSlice(mark1)); + assertEquals(LINE_3 + "\n", sourceManager.getSlice(mark1).toString()); assertEquals(4, mark2.getBeginLine()); assertEquals(fileName, mark2.getFilename()); - assertEquals(Chars.wrap(LINE_4), sourceManager.getSlice(mark2)); - } - - @Test - void testIgnore() throws IOException { - Language java = JavaLanguageModule.getInstance(); - LanguagePropertyBundle bundle = java.newPropertyBundle(); - bundle.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, true); - bundle.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, true); - Tokenizer tokenizer = java.createCpdTokenizer(bundle); - TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion()); - Tokens tokens = new Tokens(); - Tokenizer.tokenize(tokenizer, sourceCode, tokens); - - MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5); - matchAlgorithm.findMatches(); - Iterator matches = matchAlgorithm.matches(); - Match match = matches.next(); - assertFalse(matches.hasNext()); - - Iterator marks = match.iterator(); - marks.next(); - marks.next(); - marks.next(); - assertFalse(marks.hasNext()); + assertEquals(LINE_4 + "\n", sourceManager.getSlice(mark2).toString()); } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java index 9ae3acac1c..b36c86e36e 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang; import java.util.Objects; import net.sourceforge.pmd.RuleViolation; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.lang.ast.DummyNode; import net.sourceforge.pmd.lang.ast.DummyNode.DummyRootNode; import net.sourceforge.pmd.lang.ast.ParseException; @@ -45,6 +46,14 @@ public class DummyLanguageModule extends SimpleLanguageModuleBase { return (DummyLanguageModule) Objects.requireNonNull(LanguageRegistry.PMD.getLanguageByFullName(NAME)); } + @Override + public LanguagePropertyBundle newPropertyBundle() { + LanguagePropertyBundle bundle = super.newPropertyBundle(); + bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_LITERALS); + bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS); + return bundle; + } + public LanguageVersion getVersionWhereParserThrows() { return getVersion(PARSER_THROWS); } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java index eadecc970f..703582bd3d 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java @@ -6,9 +6,9 @@ package net.sourceforge.pmd.lang.cpp; import net.sourceforge.pmd.cpd.CPPTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.properties.PropertyDescriptor; import net.sourceforge.pmd.properties.PropertyFactory; @@ -19,7 +19,7 @@ public class CppLanguageModule extends CpdOnlyLanguageModuleBase { public static final PropertyDescriptor CPD_SKIP_BLOCKS = - PropertyFactory.stringProperty("cpdSkipBlocksPattern") + PropertyFactory.stringProperty("cpdSkipBlocksPattern") .defaultValue("#if 0|#endif") .desc("Specifies a start and end delimiter for CPD to completely ignore. " + "The delimiters are separated by a pipe |. The default skips code " diff --git a/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java b/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java index 5f122b8e35..bd156eb6df 100644 --- a/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java +++ b/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java @@ -4,11 +4,11 @@ package net.sourceforge.pmd.lang.cs; -import net.sourceforge.pmd.cpd.CsTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.cs.cpd.CsTokenizer; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Defines the Language module for C#. diff --git a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java b/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizer.java similarity index 99% rename from pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java rename to pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizer.java index d58ccdb0d7..22fd33a9a2 100644 --- a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java +++ b/pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizer.java @@ -1,12 +1,13 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.cs.cpd; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.Lexer; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter; diff --git a/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java b/pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java similarity index 69% rename from pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java rename to pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java index d2d3e78f80..12d5317906 100644 --- a/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java +++ b/pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java @@ -2,28 +2,22 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.cs.cpd; import static org.junit.jupiter.api.Assertions.assertThrows; -import java.util.Properties; - -import org.jetbrains.annotations.NotNull; +import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.jupiter.api.Test; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest; -import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig; import net.sourceforge.pmd.lang.ast.TokenMgrError; class CsTokenizerTest extends CpdTextComparisonTest { CsTokenizerTest() { - super(".cs"); - } - - @Override - protected String getResourcePrefix() { - return "../lang/cs/cpd/testdata"; + super("cs", ".cs"); } @Test @@ -107,23 +101,28 @@ class CsTokenizerTest extends CpdTextComparisonTest { doTest("attributes", "_ignored", skipAttributes()); } - private Properties ignoreUsings() { + private LanguagePropertyConfig ignoreUsings() { return properties(true, false, false); } - private Properties skipLiteralSequences() { + private LanguagePropertyConfig skipLiteralSequences() { return properties(false, true, false); } - private Properties skipAttributes() { + private LanguagePropertyConfig skipAttributes() { return properties(false, false, true); } - private Properties properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) { - Properties properties = new Properties(); - properties.setProperty(Tokenizer.IGNORE_USINGS, Boolean.toString(ignoreUsings)); - properties.setProperty(Tokenizer.OPTION_IGNORE_LITERAL_SEQUENCES, Boolean.toString(ignoreLiteralSequences)); - properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAttributes)); - return properties; + @Override + public @NonNull LanguagePropertyConfig defaultProperties() { + return properties(false, false, false); + } + + private LanguagePropertyConfig properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) { + return properties -> { + properties.setProperty(Tokenizer.CPD_IGNORE_IMPORTS, ignoreUsings); + properties.setProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES, ignoreLiteralSequences); + properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAttributes); + }; } } diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java b/pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java index 195eee9282..b137f4dece 100644 --- a/pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java @@ -4,10 +4,10 @@ package net.sourceforge.pmd.lang.dart; -import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for Dart diff --git a/pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java b/pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java index a2f5d1249d..803fde8ad4 100644 --- a/pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java +++ b/pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java @@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.fortran; import net.sourceforge.pmd.cpd.AnyTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for Fortran diff --git a/pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java b/pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java index 5bffc78db6..5d73c7cb5d 100644 --- a/pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java +++ b/pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java @@ -5,9 +5,9 @@ package net.sourceforge.pmd.lang.gherkin; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.gherkin.cpd.GherkinTokenizer; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for Gherkin. diff --git a/pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java b/pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java index 7ea335f9a9..979e1da007 100644 --- a/pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java +++ b/pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java @@ -5,9 +5,9 @@ package net.sourceforge.pmd.lang.go; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.go.cpd.GoTokenizer; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; public class GoLanguageModule extends CpdOnlyLanguageModuleBase { diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java index fc8e810fca..bfe683cfda 100644 --- a/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java +++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java @@ -4,10 +4,10 @@ package net.sourceforge.pmd.lang.groovy; -import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for Groovy diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java index 57b094cfd7..b2c9ff2da9 100644 --- a/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java +++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java @@ -22,8 +22,8 @@ import groovyjarjarantlr.TokenStreamException; public class GroovyTokenizer implements Tokenizer { @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokens) { - GroovyLexer lexer = new GroovyLexer(sourceCode.newReader()); + public void tokenize(TextDocument document, TokenFactory tokens) { + GroovyLexer lexer = new GroovyLexer(document.newReader()); TokenStream tokenStream = lexer.plumb(); try { diff --git a/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java b/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java index 530b2d4a12..26149963cb 100644 --- a/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java +++ b/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java @@ -19,13 +19,13 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule; public class HtmlTokenizer implements Tokenizer { @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokens) { + public void tokenize(TextDocument document, TokenFactory tokens) { HtmlLanguageModule html = HtmlLanguageModule.getInstance(); try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) { ParserTask task = new ParserTask( - sourceCode, + document, SemanticErrorReporter.noop(), LanguageProcessorRegistry.singleton(processor) ); diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizer.java similarity index 98% rename from pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java rename to pmd-java/src/main/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizer.java index e0d74c8658..1dd4192786 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizer.java @@ -1,12 +1,15 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.java.cpd; import java.util.Deque; import java.util.LinkedList; +import net.sourceforge.pmd.cpd.TokenEntry; +import net.sourceforge.pmd.cpd.TokenFactory; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.lang.TokenManager; diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokenizerTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java similarity index 63% rename from pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokenizerTest.java rename to pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java index 5f95ef8785..ebc1eea44b 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokenizerTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java @@ -1,27 +1,20 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.java.cpd; -import java.util.Properties; - -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest; +import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig; +import net.sourceforge.pmd.lang.java.JavaLanguageModule; -// TODO - enable test -@Disabled("Needs to be enabled after java-grammar changes are finalized") class JavaTokenizerTest extends CpdTextComparisonTest { JavaTokenizerTest() { - super(".java"); - } - - @Override - protected String getResourcePrefix() { - return "../lang/java/cpd/testdata"; + super(JavaLanguageModule.getInstance(), ".java"); } @Test @@ -85,32 +78,32 @@ class JavaTokenizerTest extends CpdTextComparisonTest { } - private static Properties ignoreAnnotations() { + private static LanguagePropertyConfig ignoreAnnotations() { return properties(true, false, false); } - private static Properties ignoreIdents() { + private static LanguagePropertyConfig ignoreIdents() { return properties(false, false, true); } - private static Properties ignoreLiterals() { + private static LanguagePropertyConfig ignoreLiterals() { return properties(false, true, false); } @Override - public Properties defaultProperties() { + public LanguagePropertyConfig defaultProperties() { return properties(false, false, false); } - private static Properties properties(boolean ignoreAnnotations, - boolean ignoreLiterals, - boolean ignoreIdents) { - Properties properties = new Properties(); - properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAnnotations)); - properties.setProperty(Tokenizer.IGNORE_IDENTIFIERS, Boolean.toString(ignoreIdents)); - properties.setProperty(Tokenizer.IGNORE_LITERALS, Boolean.toString(ignoreLiterals)); - return properties; + private static LanguagePropertyConfig properties(boolean ignoreAnnotations, + boolean ignoreLiterals, + boolean ignoreIdents) { + return properties -> { + properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAnnotations); + properties.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, ignoreIdents); + properties.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, ignoreLiterals); + }; } diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java index f90fe242e8..fe3c46da3f 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java @@ -4,12 +4,12 @@ package net.sourceforge.pmd.lang.ecmascript; -import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.ecmascript.ast.EcmascriptParser; +import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer; import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase; /** diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java index e3f012f8c2..7be1c12913 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java @@ -5,10 +5,10 @@ package net.sourceforge.pmd.lang.kotlin; import net.sourceforge.pmd.annotation.Experimental; -import net.sourceforge.pmd.cpd.KotlinTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase; +import net.sourceforge.pmd.lang.kotlin.cpd.KotlinTokenizer; /** * Language Module for Kotlin diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizer.java similarity index 88% rename from pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java rename to pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizer.java index 143c1f5f06..63aa140a95 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizer.java @@ -1,16 +1,16 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.kotlin.cpd; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.Lexer; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; +import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; -import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.lang.kotlin.ast.KotlinLexer; /** @@ -24,7 +24,7 @@ public class KotlinTokenizer extends AntlrTokenizer { } @Override - protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { + protected TokenManager filterTokenStream(TokenManager tokenManager) { return new KotlinTokenFilter(tokenManager); } @@ -40,7 +40,7 @@ public class KotlinTokenizer extends AntlrTokenizer { private boolean discardingPackageAndImport = false; private boolean discardingNL = false; - /* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) { + /* default */ KotlinTokenFilter(final TokenManager tokenManager) { super(tokenManager); } diff --git a/pmd-kotlin/src/test/java/net/sourceforge/pmd/cpd/KotlinTokenizerTest.java b/pmd-kotlin/src/test/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizerTest.java similarity index 80% rename from pmd-kotlin/src/test/java/net/sourceforge/pmd/cpd/KotlinTokenizerTest.java rename to pmd-kotlin/src/test/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizerTest.java index 1ab9bee77d..ea3dd542bb 100644 --- a/pmd-kotlin/src/test/java/net/sourceforge/pmd/cpd/KotlinTokenizerTest.java +++ b/pmd-kotlin/src/test/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizerTest.java @@ -1,8 +1,8 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.kotlin.cpd; import org.junit.jupiter.api.Test; @@ -14,11 +14,6 @@ class KotlinTokenizerTest extends CpdTextComparisonTest { super("kotlin", ".kt"); } - @Override - protected String getResourcePrefix() { - return "../lang/kotlin/cpd/testdata"; - } - @Test void testComments() { doTest("comment"); diff --git a/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java b/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java index b9d8fb13ac..a0688f1a97 100644 --- a/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java +++ b/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java @@ -5,8 +5,8 @@ package net.sourceforge.pmd.lang.lua; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.lua.cpd.LuaTokenizer; /** @@ -18,8 +18,15 @@ public class LuaLanguageModule extends CpdOnlyLanguageModuleBase { super(LanguageMetadata.withId("lua").name("Lua").extensions("lua")); } + @Override + public LanguagePropertyBundle newPropertyBundle() { + LanguagePropertyBundle bundle = super.newPropertyBundle(); + bundle.definePropertyDescriptor(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES); + return bundle; + } + @Override public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) { - return new LuaTokenizer(); + return new LuaTokenizer(bundle); } } diff --git a/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/cpd/LuaTokenizer.java b/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/cpd/LuaTokenizer.java index 416b36596b..6f569e711a 100644 --- a/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/cpd/LuaTokenizer.java +++ b/pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/cpd/LuaTokenizer.java @@ -4,13 +4,13 @@ package net.sourceforge.pmd.lang.lua.cpd; -import java.util.Properties; - import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.Lexer; +import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; +import net.sourceforge.pmd.lang.LanguagePropertyBundle; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.lua.ast.LuaLexer; @@ -20,20 +20,10 @@ import net.sourceforge.pmd.lang.lua.ast.LuaLexer; */ public class LuaTokenizer extends AntlrTokenizer { - private boolean ignoreLiteralSequences = false; + private final boolean ignoreLiteralSequences; - /** - * Sets the possible options for the Lua tokenizer. - * - * @param properties the properties - * @see #OPTION_IGNORE_LITERAL_SEQUENCES - */ - public void setProperties(Properties properties) { - ignoreLiteralSequences = getBooleanProperty(properties, OPTION_IGNORE_LITERAL_SEQUENCES); - } - - private boolean getBooleanProperty(final Properties properties, final String property) { - return Boolean.parseBoolean(properties.getProperty(property, Boolean.FALSE.toString())); + public LuaTokenizer(LanguagePropertyBundle bundle) { + ignoreLiteralSequences = bundle.getProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES); } @Override diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java index dc33c1970d..5e8b414d1d 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java @@ -5,8 +5,8 @@ package net.sourceforge.pmd.lang.matlab; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.matlab.cpd.MatlabTokenizer; /** diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCLanguageModule.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCLanguageModule.java index a0bb56045a..25e115c2d6 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCLanguageModule.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCLanguageModule.java @@ -4,10 +4,10 @@ package net.sourceforge.pmd.lang.objectivec; -import net.sourceforge.pmd.lang.objectivec.cpd.ObjectiveCTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; +import net.sourceforge.pmd.lang.objectivec.cpd.ObjectiveCTokenizer; /** * Defines the Language module for Objective-C diff --git a/pmd-objectivec/src/test/java/net/sourceforge/pmd/lang/objectivec/cpd/ObjectiveCTokenizerTest.java b/pmd-objectivec/src/test/java/net/sourceforge/pmd/lang/objectivec/cpd/ObjectiveCTokenizerTest.java index 720a369fce..33dffcf64b 100644 --- a/pmd-objectivec/src/test/java/net/sourceforge/pmd/lang/objectivec/cpd/ObjectiveCTokenizerTest.java +++ b/pmd-objectivec/src/test/java/net/sourceforge/pmd/lang/objectivec/cpd/ObjectiveCTokenizerTest.java @@ -15,11 +15,6 @@ class ObjectiveCTokenizerTest extends CpdTextComparisonTest { super("objectivec", ".m"); } - @Override - protected String getResourcePrefix() { - return "../lang/objectivec/cpd/testdata"; - } - @Test void testLongSample() { doTest("big_sample"); diff --git a/pmd-perl/src/main/java/net/sourceforge/pmd/lang/perl/PerlLanguageModule.java b/pmd-perl/src/main/java/net/sourceforge/pmd/lang/perl/PerlLanguageModule.java index e6305cc808..d0eba303db 100644 --- a/pmd-perl/src/main/java/net/sourceforge/pmd/lang/perl/PerlLanguageModule.java +++ b/pmd-perl/src/main/java/net/sourceforge/pmd/lang/perl/PerlLanguageModule.java @@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.perl; import net.sourceforge.pmd.cpd.AnyTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; public class PerlLanguageModule extends CpdOnlyLanguageModuleBase { diff --git a/pmd-php/src/main/java/net/sourceforge/pmd/lang/php/PhpLanguageModule.java b/pmd-php/src/main/java/net/sourceforge/pmd/lang/php/PhpLanguageModule.java index dfe3516d7a..8e6d9f33d9 100644 --- a/pmd-php/src/main/java/net/sourceforge/pmd/lang/php/PhpLanguageModule.java +++ b/pmd-php/src/main/java/net/sourceforge/pmd/lang/php/PhpLanguageModule.java @@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.php; import net.sourceforge.pmd.cpd.AnyTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for PHP diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java b/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java index ab6cdae1b2..194e8b6660 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java @@ -4,10 +4,10 @@ package net.sourceforge.pmd.lang.python; -import net.sourceforge.pmd.lang.python.cpd.PythonTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; +import net.sourceforge.pmd.lang.python.cpd.PythonTokenizer; /** * Defines the Language module for Python diff --git a/pmd-ruby/src/main/java/net/sourceforge/pmd/lang/ruby/RubyLanguageModule.java b/pmd-ruby/src/main/java/net/sourceforge/pmd/lang/ruby/RubyLanguageModule.java index 65e8ef06d7..91952318e1 100644 --- a/pmd-ruby/src/main/java/net/sourceforge/pmd/lang/ruby/RubyLanguageModule.java +++ b/pmd-ruby/src/main/java/net/sourceforge/pmd/lang/ruby/RubyLanguageModule.java @@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.ruby; import net.sourceforge.pmd.cpd.AnyTokenizer; import net.sourceforge.pmd.cpd.Tokenizer; -import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase; /** * Language implementation for Ruby. diff --git a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java index 227988325e..0a5eb4f9f5 100644 --- a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java +++ b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java @@ -38,20 +38,20 @@ public class ScalaTokenizer implements Tokenizer { } @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokenEntries) { + public void tokenize(TextDocument document, TokenFactory tokenEntries) { try { - String fullCode = sourceCode.getText().toString(); + String fullCode = document.getText().toString(); // create the input file for scala - Input.VirtualFile vf = new Input.VirtualFile(sourceCode.getDisplayName(), fullCode); + Input.VirtualFile vf = new Input.VirtualFile(document.getDisplayName(), fullCode); ScalametaTokenizer tokenizer = new ScalametaTokenizer(vf, dialect); // tokenize with a filter scala.meta.tokens.Tokens tokens = tokenizer.tokenize(); // use extensions to the standard PMD TokenManager and Filter - ScalaTokenManager scalaTokenManager = new ScalaTokenManager(tokens.iterator(), sourceCode); + ScalaTokenManager scalaTokenManager = new ScalaTokenManager(tokens.iterator(), document); ScalaTokenFilter filter = new ScalaTokenFilter(scalaTokenManager); ScalaTokenAdapter token; @@ -68,7 +68,7 @@ public class ScalaTokenizer implements Tokenizer { TokenizeException tokE = (TokenizeException) e; Position pos = tokE.pos(); throw new TokenMgrError( - pos.startLine() + 1, pos.startColumn() + 1, sourceCode.getDisplayName(), "Scalameta threw", tokE); + pos.startLine() + 1, pos.startColumn() + 1, document.getDisplayName(), "Scalameta threw", tokE); } else { throw e; } diff --git a/pmd-swift/src/main/java/net/sourceforge/pmd/lang/swift/rule/bestpractices/UnavailableFunctionRule.java b/pmd-swift/src/main/java/net/sourceforge/pmd/lang/swift/rule/bestpractices/UnavailableFunctionRule.java index 77c4700e75..77fb67f9a9 100644 --- a/pmd-swift/src/main/java/net/sourceforge/pmd/lang/swift/rule/bestpractices/UnavailableFunctionRule.java +++ b/pmd-swift/src/main/java/net/sourceforge/pmd/lang/swift/rule/bestpractices/UnavailableFunctionRule.java @@ -8,7 +8,6 @@ import java.util.List; import net.sourceforge.pmd.RuleContext; import net.sourceforge.pmd.lang.ast.AstVisitor; -import net.sourceforge.pmd.lang.swift.rule.AbstractSwiftRule; import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwAttribute; import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwAttributes; import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwCodeBlock; @@ -16,6 +15,7 @@ import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwFunctionDeclaration; import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwInitializerDeclaration; import net.sourceforge.pmd.lang.swift.ast.SwiftParser.SwStatement; import net.sourceforge.pmd.lang.swift.ast.SwiftVisitorBase; +import net.sourceforge.pmd.lang.swift.rule.AbstractSwiftRule; public class UnavailableFunctionRule extends AbstractSwiftRule {