Publish RichCharSequence instead of SharingCharSeq

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaCharStream.java

@@ -10,8 +10,6 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
-import net.sourceforge.pmd.lang.ast.internal.SharingCharSeq;
-
 /**
  * This stream buffers the whole file in memory before parsing,
  * and shares the char array between all tokens.
@@ -67,7 +65,7 @@ public class JavaCharStream extends JavaCharStreamBase {
         }
     }
 
-    public SharingCharSeq getFullText() {
+    public RichCharSequence getFullText() {
         return seq;
     }
 

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaccToken.java

@@ -13,18 +13,38 @@
 package net.sourceforge.pmd.lang.ast.impl;
 
 import net.sourceforge.pmd.lang.ast.GenericToken;
+import net.sourceforge.pmd.lang.ast.RootNode;
 
 /**
+ * A generic token implementation for JavaCC parsers. Will probably help
+ * remove those duplicated implementations that all have the same name.
+ *
+ * <p>I think the only thing important here is {@link #getStartDocumentOffset()}
+ * and {@link #getEndDocumentOffset()}. The begin/end line/column can very probably
+ * be removed from the token instances to make them lighter, which doesn't
+ * prevent them from being retrieved on-demand using the original text.
+ *
+ * <p>To make that easy we should probably link tokens to an underlying
+ * "token document", which would be the list of all tokens in the file,
+ * with the original (shared) full text. That would be available from
+ * the {@link RootNode} instances. That representation could be shared
+ * with CPD as well.
+ *
+ * TODO remove those four fields and fetch the begin/end line
+ *   dynamically from the underlying text.
+ *
  * @author Clément Fournier
  */
 public class JavaccToken implements GenericToken, java.io.Serializable {
 
+    public static final JavaccToken UNDEFINED = new JavaccToken();
+
     /**
      * The version identifier for this Serializable class.
      * Increment only if the <i>serialized</i> form of the
      * class changes.
      */
-    private static final long serialVersionUID = 1L;
+    private static final long serialVersionUID = 4L;
 
     /**
      * An integer that describes the kind of this token.  This numbering

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/RichCharSequence.java

@@ -2,14 +2,28 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.lang.ast.internal;
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ast.impl;
 
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.CharSequenceUtils;
 import org.apache.commons.lang3.StringUtils;
 
-interface RichCharSequence extends CharSequence {
+public interface RichCharSequence extends CharSequence {
+
+    @Override
+    RichCharSequence subSequence(int start, int end);
+
+
+    /** @see CharSequenceUtils#subSequence(CharSequence, int) */
+    default RichCharSequence subSequence(int start) {
+        return subSequence(start, length());
+    }
+
 
     /** @see StringUtils#indexOf(CharSequence, int) */
     default int indexOf(int searchChar) {
@@ -75,13 +89,6 @@ interface RichCharSequence extends CharSequence {
         return StringUtils.startsWith(this, prefix);
     }
 
-
-    /** @see CharSequenceUtils#subSequence(CharSequence, int) */
-    default CharSequence subSequence(int start) {
-        return subSequence(start, length());
-    }
-
-
     /** @see Pattern#matches(String, CharSequence) */
     default boolean matches(String regex) {
         return Pattern.matches(regex, this);

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SharingCharSeq.java

@@ -2,7 +2,7 @@
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.lang.ast.internal;
+package net.sourceforge.pmd.lang.ast.impl;
 
 import java.util.Objects;
 
@@ -53,7 +53,7 @@ public final class SharingCharSeq implements RichCharSequence {
     }
 
     @Override
-    public SharingCharSeq subSequence(int start, int end) {
+    public RichCharSequence subSequence(int start, int end) {
         if (start < 0 || end > count || start > end) {
             throw new IndexOutOfBoundsException("Invalid range: [" + start + "," + end + "[ not in [0," + count + "[");
         }
@@ -61,7 +61,7 @@ public final class SharingCharSeq implements RichCharSequence {
     }
 
     @Override
-    public SharingCharSeq subSequence(int start) {
+    public RichCharSequence subSequence(int start) {
         return subSequence(start, count);
     }
 

pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java

@@ -14,6 +14,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
 
 import net.sourceforge.pmd.annotation.InternalApi;
 import net.sourceforge.pmd.lang.ast.RootNode;
+import net.sourceforge.pmd.lang.ast.impl.RichCharSequence;
 import net.sourceforge.pmd.lang.java.typeresolution.ClassTypeResolver;
 
 // FUTURE Change this class to extend from SimpleJavaNode, as TypeNode is not appropriate (unless I'm wrong)
@@ -22,7 +23,7 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ro
     private ClassTypeResolver classTypeResolver;
     private List<Comment> comments;
     private Map<Integer, String> noPmdComments = Collections.emptyMap();
-    private CharSequence fileText;
+    private RichCharSequence fileText;
 
     ASTCompilationUnit(int id) {
         super(id);
@@ -41,12 +42,12 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ro
     }
 
     @Override
-    public CharSequence getText() {
+    public RichCharSequence getText() {
         return fileText;
     }
 
 
-    void setFileText(CharSequence fileText) {
+    void setFileText(RichCharSequence fileText) {
         this.fileText = fileText;
     }
 

pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/AbstractJavaNode.java

@@ -10,6 +10,7 @@ import org.checkerframework.checker.nullness.qual.NonNull;
 import net.sourceforge.pmd.lang.ast.AbstractNode;
 import net.sourceforge.pmd.lang.ast.GenericToken;
 import net.sourceforge.pmd.lang.ast.Node;
+import net.sourceforge.pmd.lang.ast.impl.RichCharSequence;
 import net.sourceforge.pmd.lang.java.symbols.table.JSymbolTable;
 import net.sourceforge.pmd.lang.symboltable.Scope;
 
@@ -20,7 +21,7 @@ abstract class AbstractJavaNode extends AbstractNode implements JavaNode {
     private JSymbolTable symbolTable;
     private Comment comment;
     private ASTCompilationUnit root;
-    private CharSequence text;
+    private RichCharSequence text;
 
     AbstractJavaNode(int id) {
         super(id);
@@ -107,7 +108,7 @@ abstract class AbstractJavaNode extends AbstractNode implements JavaNode {
 
 
     @Override
-    public CharSequence getText() {
+    public RichCharSequence getText() {
         if (text == null) {
             text = getRoot().getText().subSequence(getStartOffset(), getEndOffset());
         }

pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaNode.java

@@ -9,6 +9,7 @@ import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import net.sourceforge.pmd.lang.ast.GenericToken;
+import net.sourceforge.pmd.lang.ast.impl.RichCharSequence;
 import net.sourceforge.pmd.lang.java.symbols.table.JSymbolTable;
 import net.sourceforge.pmd.lang.symboltable.ScopedNode;
 
@@ -85,10 +86,6 @@ public interface JavaNode extends ScopedNode {
     @Override
     JavaNode jjtGetParent();
 
-
-    ASTCompilationUnit getRoot();
-
-
     GenericToken jjtGetFirstToken();
 
 
@@ -119,7 +116,7 @@ public interface JavaNode extends ScopedNode {
     int getEndOffset();
 
 
-    CharSequence getText();
+    RichCharSequence getText();
 
 
     /**