Fix build

justfile

@@ -20,6 +20,9 @@ alias i := install
 cleanInstallEverything *FLAGS:
    mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
 
+installEverything *FLAGS:
+   mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
+
 testCore *FLAGS:
    mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}}
 
@@ -46,6 +49,6 @@ lintChanged DIFF="master" *FLAGS="":
 lint projects="pmd-java":
   mvnd checkstyle:check pmd:check -pl {{projects}} -fae
 
-lintAll:
+lintAll *FLAGS:
-  mvnd checkstyle:check pmd:check -fae
+  mvnd checkstyle:check pmd:check -fae {{FLAGS}}
 

pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml

@@ -8,7 +8,7 @@
     <target name="all" depends="testBasic"/>
 
 	<target name="testBasic">
-        <cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="cpddummy">
+        <cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="dummy">
           <fileset dir="${pmd.home}/src/main/">
             <include name="**/*.java"/>
           </fileset>

pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java

@@ -4,13 +4,13 @@
 
 package net.sourceforge.pmd.lang.apex;
 
-import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.lang.Language;
 import net.sourceforge.pmd.lang.LanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguageProcessor;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
 
 import apex.jorje.services.Version;
 

pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java

@@ -4,32 +4,60 @@
 
 package net.sourceforge.pmd.lang.apex.cpd;
 
+import java.io.IOException;
 import java.util.Locale;
 
-import org.antlr.v4.runtime.CharStream;
+import org.antlr.runtime.ANTLRReaderStream;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.Lexer;
+import org.antlr.runtime.Token;
 
+import net.sourceforge.pmd.cpd.TokenFactory;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.lang.apex.ApexJorjeLogging;
 import net.sourceforge.pmd.lang.apex.ApexLanguageProperties;
-import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.document.TextDocument;
+
+import apex.jorje.parser.impl.ApexLexer;
+
+public class ApexTokenizer implements Tokenizer {
 
-public class ApexTokenizer extends AntlrTokenizer {
     private final boolean caseSensitive;
 
     public ApexTokenizer(ApexLanguageProperties properties) {
         this.caseSensitive = properties.getProperty(Tokenizer.CPD_CASE_SENSITIVE);
+        ApexJorjeLogging.disableLogging();
     }
 
     @Override
-    protected String getImage(AntlrToken token) {
+    public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException {
-        if (caseSensitive) {
-            return token.getImage();
-        }
-        return token.getImage().toLowerCase(Locale.ROOT);
-    }
 
+        ANTLRStringStream ass = new ANTLRReaderStream(document.newReader());
+        ApexLexer lexer = new ApexLexer(ass) {
             @Override
-    protected org.antlr.v4.runtime.Lexer getLexerForSource(CharStream charStream) {
+            public void emitErrorMessage(String msg) {
-        return new com.nawforce.runtime.parsers.ApexLexer(charStream);
+                throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null);
+            }
+        };
+
+        Token token = lexer.nextToken();
+
+        while (token.getType() != Token.EOF) {
+            if (token.getChannel() != Lexer.HIDDEN) {
+                String tokenText = token.getText();
+                if (!caseSensitive) {
+                    tokenText = tokenText.toLowerCase(Locale.ROOT);
+                }
+                tokenEntries.recordToken(
+                    tokenText,
+                    token.getLine(),
+                    token.getCharPositionInLine() + 1,
+                    token.getLine(),
+                    token.getCharPositionInLine() + tokenText.length() + 1
+                );
+            }
+            token = lexer.nextToken();
+        }
     }
 }

pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java

@@ -124,7 +124,7 @@ public class CpdCommand extends AbstractAnalysisPmdSubcommand {
         
         final CPDConfiguration configuration = toConfiguration();
 
-        try (CpdAnalysis cpd = new CpdAnalysis(configuration)){
+        try (CpdAnalysis cpd = new CpdAnalysis(configuration)) {
 
             MutableBoolean hasViolations = new MutableBoolean();
             cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty()));

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java

@@ -62,8 +62,8 @@ public class AnyTokenizer implements Tokenizer {
     }
 
     @Override
-    public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
+    public void tokenize(TextDocument document, TokenFactory tokens) {
-        Chars text = sourceCode.getText();
+        Chars text = document.getText();
         Matcher matcher = pattern.matcher(text);
         int lineNo = 1;
         int lastLineStart = 0;

pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java

@@ -62,11 +62,11 @@ import javax.swing.table.TableModel;
 
 import net.sourceforge.pmd.PMDVersion;
 import net.sourceforge.pmd.cpd.renderer.CPDReportRenderer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.Language;
 import net.sourceforge.pmd.lang.LanguageModuleBase.LanguageMetadata;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 public class GUI implements CPDListener {
 

pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java

@@ -26,10 +26,8 @@ public interface Tokenizer {
     PropertyDescriptor<Boolean> CPD_ANONYMIZE_IDENTIFIERS =
         PropertyFactory.booleanProperty("cpdAnonymizeIdentifiers")
                        .defaultValue(false)
-                       .desc("Anonymize identifiers. They are still part of the token stream but all literals appear to have the same value.")
+                       .desc("Anonymize identifiers. They are still part of the token stream but all identifiers appear to have the same value.")
                        .build();
-
-
     PropertyDescriptor<Boolean> CPD_IGNORE_IMPORTS =
         PropertyFactory.booleanProperty("cpdIgnoreImports")
                        .defaultValue(true)
@@ -41,48 +39,17 @@ public interface Tokenizer {
                        .defaultValue(false)
                        .desc("Ignore metadata such as Java annotations or C# attributes.")
                        .build();
-
-
     PropertyDescriptor<Boolean> CPD_CASE_SENSITIVE =
         PropertyFactory.booleanProperty("cpdCaseSensitive")
-                       .defaultValue(true)
+                       .defaultValue(false)
                        .desc("Whether CPD should ignore the case of tokens. Affects all tokens.")
                        .build();
 
 
-    String IGNORE_LITERALS = "ignore_literals";
+    @Deprecated // TODO what to do with this?
-    String IGNORE_IDENTIFIERS = "ignore_identifiers";
-    String IGNORE_ANNOTATIONS = "ignore_annotations";
-
-    /**
-     * Ignore sequences of literals (e.g, <code>0,0,0,0...</code>).
-     */
-    String OPTION_IGNORE_LITERAL_SEQUENCES = "net.sourceforge.pmd.cpd.Tokenizer.skipLiteralSequences";
-    /**
-     * Ignore using directives in C#. The default value is <code>false</code>.
-     */
-    String IGNORE_USINGS = "ignore_usings";
-
-    /**
-     * Enables or disabled skipping of blocks like a pre-processor. It is a
-     * boolean property. The default value is <code>true</code>.
-     *
-     * @see #OPTION_SKIP_BLOCKS_PATTERN
-     */
-    String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks";
-    /**
-     * Configures the pattern, to find the blocks to skip. It is a string
-     * property and contains of two parts, separated by {@code |}. The first
-     * part is the start pattern, the second part is the ending pattern. Default
-     * value is "{@code #if 0|#endif}".
-     *
-     * @see #DEFAULT_SKIP_BLOCKS_PATTERN
-     */
-    String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern";
-
     String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif";
 
-    void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException;
+    void tokenize(TextDocument document, TokenFactory tokens) throws IOException;
 
     static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException {
         try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) {

pmd-core/src/main/java/net/sourceforge/pmd/internal/LanguageServiceBase.java

@@ -1,68 +0,0 @@
-/*
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.internal;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.ServiceConfigurationError;
-import java.util.ServiceLoader;
-import java.util.Set;
-import java.util.TreeSet;
-
-import net.sourceforge.pmd.annotation.InternalApi;
-
-@InternalApi
-public abstract class LanguageServiceBase<T> {
-
-    protected interface NameExtractor<T> {
-        String getName(T language);
-    }
-
-    protected final Set<T> languages;
-    protected final Map<String, T> languagesByName;
-    protected final Map<String, T> languagesByTerseName;
-
-    protected LanguageServiceBase(final Class<T> serviceType, final Comparator<T> comparator,
-            final NameExtractor<T> nameExtractor, final NameExtractor<T> terseNameExtractor) {
-        Set<T> sortedLangs = new TreeSet<>(comparator);
-        // Use current class' classloader instead of the threads context classloader, see https://github.com/pmd/pmd/issues/1788
-        ServiceLoader<T> languageLoader = ServiceLoader.load(serviceType, getClass().getClassLoader());
-        Iterator<T> iterator = languageLoader.iterator();
-
-        while (true) {
-            // this loop is weird, but both hasNext and next may throw ServiceConfigurationError,
-            // it's more robust that way
-            try {
-                if (iterator.hasNext()) {
-                    T language = iterator.next();
-                    sortedLangs.add(language);
-                } else {
-                    break;
-                }
-            } catch (UnsupportedClassVersionError | ServiceConfigurationError e) {
-                // Some languages require java8 and are therefore only available
-                // if java8 or later is used as runtime.
-                System.err.println("Ignoring language for PMD: " + e.toString());
-            }
-        }
-
-        // using a linked hash map to maintain insertion order
-        languages = Collections.unmodifiableSet(new LinkedHashSet<>(sortedLangs));
-
-        // TODO there may be languages with duplicate names
-        Map<String, T> byName = new LinkedHashMap<>();
-        Map<String, T> byTerseName = new LinkedHashMap<>();
-        for (T language : sortedLangs) {
-            byName.put(nameExtractor.getName(language), language);
-            byTerseName.put(terseNameExtractor.getName(language), language);
-        }
-        languagesByName = Collections.unmodifiableMap(byName);
-        languagesByTerseName = Collections.unmodifiableMap(byTerseName);
-    }
-}

pmd-core/src/main/java/net/sourceforge/pmd/lang/impl/CpdOnlyLanguageModuleBase.java

@@ -2,9 +2,11 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.lang;
+package net.sourceforge.pmd.lang.impl;
 
 import net.sourceforge.pmd.cpd.Tokenizer;
+import net.sourceforge.pmd.lang.LanguageModuleBase;
+import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 
 /**
  * Base class for language modules that only support CPD and not PMD.

pmd-core/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java

@@ -1,4 +1,4 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
@@ -13,12 +13,10 @@ import java.util.Iterator;
 
 import org.junit.jupiter.api.Test;
 
+import net.sourceforge.pmd.lang.DummyLanguageModule;
 import net.sourceforge.pmd.lang.Language;
-import net.sourceforge.pmd.lang.LanguagePropertyBundle;
-import net.sourceforge.pmd.lang.document.Chars;
 import net.sourceforge.pmd.lang.document.TextDocument;
 import net.sourceforge.pmd.lang.document.TextFile;
-import net.sourceforge.pmd.lang.java.JavaLanguageModule;
 
 class MatchAlgorithmTest {
 
@@ -38,15 +36,15 @@ class MatchAlgorithmTest {
 
     @Test
     void testSimple() throws IOException {
-        Language java = JavaLanguageModule.getInstance();
+        Language dummy = DummyLanguageModule.getInstance();
-        Tokenizer tokenizer = java.createCpdTokenizer(java.newPropertyBundle());
+        Tokenizer tokenizer = dummy.createCpdTokenizer(dummy.newPropertyBundle());
-        String fileName = "Foo.java";
+        String fileName = "Foo.dummy";
-        TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, java.getDefaultVersion());
+        TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, dummy.getDefaultVersion());
         SourceManager sourceManager = new SourceManager(listOf(textFile));
         Tokens tokens = new Tokens();
         TextDocument sourceCode = sourceManager.get(textFile);
         Tokenizer.tokenize(tokenizer, sourceCode, tokens);
-        assertEquals(41, tokens.size());
+        assertEquals(44, tokens.size());
 
         MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
         matchAlgorithm.findMatches();
@@ -61,34 +59,10 @@ class MatchAlgorithmTest {
 
         assertEquals(3, mark1.getBeginLine());
         assertEquals(fileName, mark1.getFilename());
-        assertEquals(Chars.wrap(LINE_3), sourceManager.getSlice(mark1));
+        assertEquals(LINE_3 + "\n", sourceManager.getSlice(mark1).toString());
 
         assertEquals(4, mark2.getBeginLine());
         assertEquals(fileName, mark2.getFilename());
-        assertEquals(Chars.wrap(LINE_4), sourceManager.getSlice(mark2));
+        assertEquals(LINE_4 + "\n", sourceManager.getSlice(mark2).toString());
-    }
-
-    @Test
-    void testIgnore() throws IOException {
-        Language java = JavaLanguageModule.getInstance();
-        LanguagePropertyBundle bundle = java.newPropertyBundle();
-        bundle.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, true);
-        bundle.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, true);
-        Tokenizer tokenizer = java.createCpdTokenizer(bundle);
-        TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion());
-        Tokens tokens = new Tokens();
-        Tokenizer.tokenize(tokenizer, sourceCode, tokens);
-
-        MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
-        matchAlgorithm.findMatches();
-        Iterator<Match> matches = matchAlgorithm.matches();
-        Match match = matches.next();
-        assertFalse(matches.hasNext());
-
-        Iterator<Mark> marks = match.iterator();
-        marks.next();
-        marks.next();
-        marks.next();
-        assertFalse(marks.hasNext());
     }
 }

pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java

@@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang;
 import java.util.Objects;
 
 import net.sourceforge.pmd.RuleViolation;
+import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.lang.ast.DummyNode;
 import net.sourceforge.pmd.lang.ast.DummyNode.DummyRootNode;
 import net.sourceforge.pmd.lang.ast.ParseException;
@@ -45,6 +46,14 @@ public class DummyLanguageModule extends SimpleLanguageModuleBase {
         return (DummyLanguageModule) Objects.requireNonNull(LanguageRegistry.PMD.getLanguageByFullName(NAME));
     }
 
+    @Override
+    public LanguagePropertyBundle newPropertyBundle() {
+        LanguagePropertyBundle bundle = super.newPropertyBundle();
+        bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_LITERALS);
+        bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS);
+        return bundle;
+    }
+
     public LanguageVersion getVersionWhereParserThrows() {
         return getVersion(PARSER_THROWS);
     }

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java

@@ -6,9 +6,9 @@ package net.sourceforge.pmd.lang.cpp;
 
 import net.sourceforge.pmd.cpd.CPPTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.properties.PropertyDescriptor;
 import net.sourceforge.pmd.properties.PropertyFactory;
 

pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java

@@ -4,11 +4,11 @@
 
 package net.sourceforge.pmd.lang.cs;
 
-import net.sourceforge.pmd.cpd.CsTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.cs.cpd.CsTokenizer;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 /**
  * Defines the Language module for C#.

pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizer.java

@@ -1,12 +1,13 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.cs.cpd;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
+import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;

pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java

@@ -2,28 +2,22 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.cs.cpd;
 
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import java.util.Properties;
+import org.checkerframework.checker.nullness.qual.NonNull;
-
-import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.Test;
 
+import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
-import net.sourceforge.pmd.lang.LanguagePropertyBundle;
+import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 class CsTokenizerTest extends CpdTextComparisonTest {
 
     CsTokenizerTest() {
-        super(".cs");
+        super("cs", ".cs");
-    }
-
-    @Override
-    protected String getResourcePrefix() {
-        return "../lang/cs/cpd/testdata";
     }
 
     @Test
@@ -107,23 +101,28 @@ class CsTokenizerTest extends CpdTextComparisonTest {
         doTest("attributes", "_ignored", skipAttributes());
     }
 
-    private Properties ignoreUsings() {
+    private LanguagePropertyConfig ignoreUsings() {
         return properties(true, false, false);
     }
 
-    private Properties skipLiteralSequences() {
+    private LanguagePropertyConfig skipLiteralSequences() {
         return properties(false, true, false);
     }
 
-    private Properties skipAttributes() {
+    private LanguagePropertyConfig skipAttributes() {
         return properties(false, false, true);
     }
 
-    private Properties properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
+    @Override
-        Properties properties = new Properties();
+    public @NonNull LanguagePropertyConfig defaultProperties() {
-        properties.setProperty(Tokenizer.IGNORE_USINGS, Boolean.toString(ignoreUsings));
+        return properties(false, false, false);
-        properties.setProperty(Tokenizer.OPTION_IGNORE_LITERAL_SEQUENCES, Boolean.toString(ignoreLiteralSequences));
+    }
-        properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAttributes));
+
-        return properties;
+    private LanguagePropertyConfig properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
+        return properties -> {
+            properties.setProperty(Tokenizer.CPD_IGNORE_IMPORTS, ignoreUsings);
+            properties.setProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES, ignoreLiteralSequences);
+            properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAttributes);
+        };
     }
 }

pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java

@@ -4,10 +4,10 @@
 
 package net.sourceforge.pmd.lang.dart;
 
-import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
+import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 /**
  * Language implementation for Dart

pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java

@@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.fortran;
 
 import net.sourceforge.pmd.cpd.AnyTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 /**
  * Language implementation for Fortran

pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java

@@ -5,9 +5,9 @@
 package net.sourceforge.pmd.lang.gherkin;
 
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.gherkin.cpd.GherkinTokenizer;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 /**
  * Language implementation for Gherkin.

pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java

@@ -5,9 +5,9 @@
 package net.sourceforge.pmd.lang.go;
 
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.go.cpd.GoTokenizer;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 public class GoLanguageModule extends CpdOnlyLanguageModuleBase {
 

pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java

@@ -4,10 +4,10 @@
 
 package net.sourceforge.pmd.lang.groovy;
 
-import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
+import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 
 /**
  * Language implementation for Groovy

pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java

@@ -22,8 +22,8 @@ import groovyjarjarantlr.TokenStreamException;
 public class GroovyTokenizer implements Tokenizer {
 
     @Override
-    public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
+    public void tokenize(TextDocument document, TokenFactory tokens) {
-        GroovyLexer lexer = new GroovyLexer(sourceCode.newReader());
+        GroovyLexer lexer = new GroovyLexer(document.newReader());
         TokenStream tokenStream = lexer.plumb();
 
         try {

pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java

@@ -19,13 +19,13 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule;
 public class HtmlTokenizer implements Tokenizer {
 
     @Override
-    public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
+    public void tokenize(TextDocument document, TokenFactory tokens) {
         HtmlLanguageModule html = HtmlLanguageModule.getInstance();
 
         try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) {
 
             ParserTask task = new ParserTask(
-                sourceCode,
+                document,
                 SemanticErrorReporter.noop(),
                 LanguageProcessorRegistry.singleton(processor)
             );

pmd-java/src/main/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizer.java

@@ -1,12 +1,15 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.java.cpd;
 
 import java.util.Deque;
 import java.util.LinkedList;
 
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.TokenFactory;
+import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;

pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java

@@ -1,27 +1,20 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.java.cpd;
 
-import java.util.Properties;
-
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
+import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
+import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
+import net.sourceforge.pmd.lang.java.JavaLanguageModule;
 
-// TODO - enable test
-@Disabled("Needs to be enabled after java-grammar changes are finalized")
 class JavaTokenizerTest extends CpdTextComparisonTest {
 
     JavaTokenizerTest() {
-        super(".java");
+        super(JavaLanguageModule.getInstance(), ".java");
-    }
-
-    @Override
-    protected String getResourcePrefix() {
-        return "../lang/java/cpd/testdata";
     }
 
     @Test
@@ -85,32 +78,32 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
     }
 
 
-    private static Properties ignoreAnnotations() {
+    private static LanguagePropertyConfig ignoreAnnotations() {
         return properties(true, false, false);
     }
 
-    private static Properties ignoreIdents() {
+    private static LanguagePropertyConfig ignoreIdents() {
         return properties(false, false, true);
     }
 
-    private static Properties ignoreLiterals() {
+    private static LanguagePropertyConfig ignoreLiterals() {
         return properties(false, true, false);
     }
 
 
     @Override
-    public Properties defaultProperties() {
+    public LanguagePropertyConfig defaultProperties() {
         return properties(false, false, false);
     }
 
-    private static Properties properties(boolean ignoreAnnotations,
+    private static LanguagePropertyConfig properties(boolean ignoreAnnotations,
                                                      boolean ignoreLiterals,
                                                      boolean ignoreIdents) {
-        Properties properties = new Properties();
+        return properties -> {
-        properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAnnotations));
+            properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAnnotations);
-        properties.setProperty(Tokenizer.IGNORE_IDENTIFIERS, Boolean.toString(ignoreIdents));
+            properties.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, ignoreIdents);
-        properties.setProperty(Tokenizer.IGNORE_LITERALS, Boolean.toString(ignoreLiterals));
+            properties.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, ignoreLiterals);
-        return properties;
+        };
     }
 
 

pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java

@@ -4,12 +4,12 @@
 
 package net.sourceforge.pmd.lang.ecmascript;
 
-import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.lang.Language;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.ecmascript.ast.EcmascriptParser;
+import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
 import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
 
 /**

pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java

@@ -5,10 +5,10 @@
 package net.sourceforge.pmd.lang.kotlin;
 
 import net.sourceforge.pmd.annotation.Experimental;
-import net.sourceforge.pmd.cpd.KotlinTokenizer;
 import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
+import net.sourceforge.pmd.lang.kotlin.cpd.KotlinTokenizer;
 
 /**
  * Language Module for Kotlin

pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizer.java

@@ -1,16 +1,16 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.kotlin.cpd;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
 import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
+import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
-import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 import net.sourceforge.pmd.lang.kotlin.ast.KotlinLexer;
 
 /**
@@ -24,7 +24,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
     }
 
     @Override
-    protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
+    protected TokenManager<AntlrToken> filterTokenStream(TokenManager<AntlrToken> tokenManager) {
         return new KotlinTokenFilter(tokenManager);
     }
 
@@ -40,7 +40,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
         private boolean discardingPackageAndImport = false;
         private boolean discardingNL = false;
 
-        /* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) {
+        /* default */ KotlinTokenFilter(final TokenManager<AntlrToken> tokenManager) {
             super(tokenManager);
         }
 

pmd-kotlin/src/test/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizerTest.java

@@ -1,8 +1,8 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd;
+package net.sourceforge.pmd.lang.kotlin.cpd;
 
 import org.junit.jupiter.api.Test;
 
@@ -14,11 +14,6 @@ class KotlinTokenizerTest extends CpdTextComparisonTest {
         super("kotlin", ".kt");
     }
 
-    @Override
-    protected String getResourcePrefix() {
-        return "../lang/kotlin/cpd/testdata";
-    }
-
     @Test
     void testComments() {
         doTest("comment");

pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java

@@ -5,8 +5,8 @@
 package net.sourceforge.pmd.lang.lua;
 
 import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
+import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
 import net.sourceforge.pmd.lang.lua.cpd.LuaTokenizer;
 
 /**
@@ -18,8 +18,15 @@ public class LuaLanguageModule extends CpdOnlyLanguageModuleBase {
         super(LanguageMetadata.withId("lua").name("Lua").extensions("lua"));
     }
 
+    @Override
+    public LanguagePropertyBundle newPropertyBundle() {
+        LanguagePropertyBundle bundle = super.newPropertyBundle();
+        bundle.definePropertyDescriptor(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES);
+        return bundle;
+    }
+
     @Override
     public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) {
-        return new LuaTokenizer();
+        return new LuaTokenizer(bundle);
     }
 }