[core] Move AntlrTokenizer and JavaCCTokenizer into impl packages

docs/pages/pmd/devdocs/major_contributions/adding_new_cpd_language.md

@@ -25,8 +25,9 @@ All you need to do is follow this few steps:
 
 2. Create a Tokenizer
     
-    - For Antlr grammars you can take the grammar from [here](https://github.com/antlr/grammars-v4)  and extend [AntlrTokenizer](https://github.com/pmd/pmd/blob/master/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java)  taking Go as an example
+    - For Antlr grammars you can take the grammar from [here](https://github.com/antlr/grammars-v4) and
-    
+      extend [AntlrTokenizer](https://github.com/pmd/pmd/blob/master/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrTokenizer.java) 
+      taking Go as an example
     
     ```java 
       public class GoTokenizer extends AntlrTokenizer {    
@@ -38,7 +39,9 @@ All you need to do is follow this few steps:
       }
     ```
     
-    - For JavaCC grammars you should subclass [JavaCCTokenizer](https://github.com/pmd/pmd/blob/master/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java) which has many examples you could follow, you should also take the [Python implementation](https://github.com/pmd/pmd/blob/master/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java) as reference
+    - For JavaCC grammars you should subclass [JavaCCTokenizer](https://github.com/pmd/pmd/blob/master/pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/JavaCCTokenizer.java)
+      which has many examples you could follow, you should also take the
+      [Python implementation](https://github.com/pmd/pmd/blob/master/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java) as reference
     - For any other scenario you can use [AnyTokenizer](https://github.com/pmd/pmd/blob/master/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java)
 
  If you're using Antlr or JavaCC, update the pom.xml of your submodule to use the appropriate ant wrapper. See `pmd-go/pom.xml` and `pmd-python/pom.xml` for examples.

docs/pages/release_notes.md

@@ -39,7 +39,12 @@ for all.</p>
 This section lists the most important changes from the last release candidate.
 The remaining section describe the complete release notes for 7.0.0.
 
-Fixed Issues:
+#### API Changes
+* Moved the two classes {% jdoc core::cpd.impl.AntlrTokenizer %} and {% jdoc core::cpd.impl.JavaCCTokenizer %} from
+  `internal` package into package {% jdoc_package core::cpd.impl %}. These two classes are part of the API and
+  are base classes for CPD language implementations.
+
+#### Fixed Issues:
 * java-codestyle
   * [#4273](https://github.com/pmd/pmd/issues/4273): \[java] CommentDefaultAccessModifier ignoredAnnotations should include "org.junit.jupiter.api.extension.RegisterExtension" by default
 * java-errorprone

docs/pages/release_notes_pmd7.md

@@ -750,6 +750,10 @@ until the next major release, but it is recommended to stop using them.
   * {% jdoc core::util.IteratorUtil %}
   * {% jdoc core::util.StringUtil %}
 
+* Moved the two classes {% jdoc core::cpd.impl.AntlrTokenizer %} and {% jdoc core::cpd.impl.JavaCCTokenizer %} from
+  `internal` package into package {% jdoc_package core::cpd.impl %}. These two classes are part of the API and
+  are base classes for CPD language implementations.
+
 ### XPath 3.1 support
 
 Support for XPath versions 1.0, 1.0-compatibility was removed, support for XPath 2.0 is deprecated. The default

pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/AntlrTokenizer.java

@@ -2,7 +2,7 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd.internal;
+package net.sourceforge.pmd.cpd.impl;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;

pmd-core/src/main/java/net/sourceforge/pmd/cpd/impl/JavaCCTokenizer.java

@@ -2,7 +2,7 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd.internal;
+package net.sourceforge.pmd.cpd.impl;
 
 import java.io.IOException;
 

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java

@@ -10,7 +10,7 @@ import java.util.List;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
 import net.sourceforge.pmd.lang.document.TextDocument;
 

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import java.util.Properties;
 import java.util.regex.Pattern;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;

pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java

@@ -9,7 +9,7 @@ import java.util.Properties;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;

pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;

pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/cpd/GherkinTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.gherkin.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.lang.gherkin.ast.GherkinLexer;
 
 /**

pmd-go/src/main/java/net/sourceforge/pmd/cpd/GoTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.lang.go.ast.GolangLexer;
 
 public class GoTokenizer extends AntlrTokenizer {

pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java

@@ -9,7 +9,7 @@ import java.util.Deque;
 import java.util.LinkedList;
 import java.util.Properties;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;

pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;

pmd-lua/src/main/java/net/sourceforge/pmd/cpd/LuaTokenizer.java

@@ -9,7 +9,7 @@ import java.util.Properties;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;

pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;

pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java

@@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
 
 import java.util.Properties;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java

@@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
 
 import java.util.regex.Pattern;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.lang.swift.ast.SwiftLexer;
 
 /**

pmd-tsql/src/main/java/net/sourceforge/pmd/lang/tsql/cpd/TSqlTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.tsql.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.lang.tsql.ast.TSqlLexer;
 
 public class TSqlTokenizer extends AntlrTokenizer {

pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java

@@ -4,7 +4,7 @@
 
 package net.sourceforge.pmd.cpd;
 
-import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.cpd.impl.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeTranslator;

pmd-xml/src/main/java/net/sourceforge/pmd/xml/cpd/XmlTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.xml.cpd;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
 
-import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
+import net.sourceforge.pmd.cpd.impl.AntlrTokenizer;
 import net.sourceforge.pmd.lang.xml.antlr4.XMLLexer;
 
 public class XmlTokenizer extends AntlrTokenizer {