Fixups for #2963

- Add documentation for CPD cli
- Add javadoc for the tokenizer options
- GenericToken::getKind is @Experimental

docs/pages/pmd/userdocs/cpd/cpd.md

@@ -119,7 +119,7 @@ Novice as much as advanced readers may want to [read on on Refactoring Guru](htt
     {% include custom/cli_option_row.html options="--ignore-literal-sequences"
                description="Ignore sequences of literals (common e.g. in list initializers)"
                default="false"
-               languages="C#"
+               languages="C#, C++"
     %}
     {% include custom/cli_option_row.html options="--ignore-usings"
                description="Ignore `using` directives in C# when comparing text"

docs/pages/release_notes.md

@@ -38,7 +38,7 @@ This is a {{ site.pmd.release_type }} release.
 
 ### API Changes
 
-#### Deprecated API
+#### Deprecated APIs
 
 *   {% jdoc !!java::lang.java.ast.ASTPackageDeclaration#getPackageNameImage() %},
     {% jdoc !!java::lang.java.ast.ASTTypeParameter#getParameterName() %}
@@ -48,7 +48,15 @@ This is a {{ site.pmd.release_type }} release.
     and {% jdoc !!java::lang.java.ast.ASTClassOrInterfaceBody#isEnumChild() %},
     refs [#905](https://github.com/pmd/pmd/issues/905)
 
-#### Internal API
+#### Experimental APIs
+
+*   The method {% jdoc !!core::lang.ast.GenericToken#getKind() %} has been added as experimental. This
+    unifies the token interface for both JavaCC and Antlr. The already existing method
+    {% jdoc !!core::cpd.token.AntlrToken#getKind() %} is therefore experimental as well. The
+    returned constant depends on the actual language and might change whenever the grammar
+    of the language is changed.
+
+#### Internal APIs
 
 Those APIs are not intended to be used by clients, and will be hidden or removed with PMD 7.0.0.
 You can identify them with the `@InternalApi` annotation. You'll also get a deprecation warning.

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java

@@ -10,6 +10,7 @@ import java.util.regex.Pattern;
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.Token;
 
+import net.sourceforge.pmd.annotation.Experimental;
 import net.sourceforge.pmd.lang.ast.GenericToken;
 
 /**
@@ -133,6 +134,7 @@ public class AntlrToken implements GenericToken {
     }
 
     @Override
+    @Experimental
     public int getKind() {
         return token.getType();
     }

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java

@@ -56,8 +56,13 @@ public interface GenericToken {
 
     /**
      * Gets a unique integer representing the kind of token this is.
-     *
      * The semantics of this kind depend on the language.
+     *
+     * <p><strong>Note:</strong> This is an experimental API.
+     *
+     * <p>The returned constants can be looked up in the language's "*ParserConstants",
+     * e.g. CppParserConstants or JavaParserConstants. These constants are considered
+     * internal API and may change at any time when the language's grammar is changed.
      */
     @Experimental
     int getKind();

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -49,7 +49,8 @@ public class CPPTokenizer extends JavaCCTokenizer {
                 skipBlocksEnd = split[1];
             }
         }
-        ignoreLiteralSequences = Boolean.parseBoolean(properties.getProperty(OPTION_IGNORE_LITERAL_SEQUENCES, "false"));
+        ignoreLiteralSequences = Boolean.parseBoolean(properties.getProperty(OPTION_IGNORE_LITERAL_SEQUENCES,
+                Boolean.FALSE.toString()));
     }
 
     private String maybeSkipBlocks(String test) throws IOException {

pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java

@@ -21,9 +21,17 @@ public class CsTokenizer extends AntlrTokenizer {
     private boolean ignoreUsings = false;
     private boolean ignoreLiteralSequences = false;
 
+    /**
+     * Sets the possible options for the C# tokenizer.
+     *
+     * @param properties the properties
+     * @see #IGNORE_USINGS
+     * @see #OPTION_IGNORE_LITERAL_SEQUENCES
+     */
     public void setProperties(Properties properties) {
-        ignoreUsings = Boolean.parseBoolean(properties.getProperty(IGNORE_USINGS, "false"));
-        ignoreLiteralSequences = Boolean.parseBoolean(properties.getProperty(OPTION_IGNORE_LITERAL_SEQUENCES, "false"));
+        ignoreUsings = Boolean.parseBoolean(properties.getProperty(IGNORE_USINGS, Boolean.FALSE.toString()));
+        ignoreLiteralSequences = Boolean.parseBoolean(properties.getProperty(OPTION_IGNORE_LITERAL_SEQUENCES,
+            Boolean.FALSE.toString()));
     }
 
     public void setIgnoreUsings(boolean ignoreUsings) {