diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/AncestorOrSelfIterator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/AncestorOrSelfIterator.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/AncestorOrSelfIterator.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/AncestorOrSelfIterator.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/AxisStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/AxisStream.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/AxisStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/AxisStream.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/DescendantOrSelfIterator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/DescendantOrSelfIterator.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/DescendantOrSelfIterator.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/DescendantOrSelfIterator.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/Filtermap.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/Filtermap.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/Filtermap.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/Filtermap.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/IteratorBasedNStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/IteratorBasedNStream.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/IteratorBasedNStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/IteratorBasedNStream.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaCharStream.java index 0b82067eac..7cf6361ce5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaCharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaCharStream.java @@ -10,8 +10,6 @@ import java.io.StringReader; import org.apache.commons.io.IOUtils; -import net.sourceforge.pmd.lang.ast.internal.SharingCharSeq; - /** * This stream buffers the whole file in memory before parsing, * and shares the char array between all tokens. @@ -67,7 +65,7 @@ public class JavaCharStream extends JavaCharStreamBase { } } - public SharingCharSeq getFullText() { + public RichCharSequence getFullText() { return seq; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaccToken.java index e7528b2be0..415a008d36 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/JavaccToken.java @@ -13,18 +13,38 @@ package net.sourceforge.pmd.lang.ast.impl; import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.ast.RootNode; /** + * A generic token implementation for JavaCC parsers. Will probably help + * remove those duplicated implementations that all have the same name. + * + *

I think the only thing important here is {@link #getStartDocumentOffset()} + * and {@link #getEndDocumentOffset()}. The begin/end line/column can very probably + * be removed from the token instances to make them lighter, which doesn't + * prevent them from being retrieved on-demand using the original text. + * + *

To make that easy we should probably link tokens to an underlying + * "token document", which would be the list of all tokens in the file, + * with the original (shared) full text. That would be available from + * the {@link RootNode} instances. That representation could be shared + * with CPD as well. + * + * TODO remove those four fields and fetch the begin/end line + * dynamically from the underlying text. + * * @author Clément Fournier */ public class JavaccToken implements GenericToken, java.io.Serializable { + public static final JavaccToken UNDEFINED = new JavaccToken(); + /** * The version identifier for this Serializable class. * Increment only if the serialized form of the * class changes. */ - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 4L; /** * An integer that describes the kind of this token. This numbering diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/RichCharSequence.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/RichCharSequence.java similarity index 88% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/RichCharSequence.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/RichCharSequence.java index 524e75cc77..1900a0fc9f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/RichCharSequence.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/RichCharSequence.java @@ -2,14 +2,28 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.internal; +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl; import java.util.regex.Pattern; import org.apache.commons.lang3.CharSequenceUtils; import org.apache.commons.lang3.StringUtils; -interface RichCharSequence extends CharSequence { +public interface RichCharSequence extends CharSequence { + + @Override + RichCharSequence subSequence(int start, int end); + + + /** @see CharSequenceUtils#subSequence(CharSequence, int) */ + default RichCharSequence subSequence(int start) { + return subSequence(start, length()); + } + /** @see StringUtils#indexOf(CharSequence, int) */ default int indexOf(int searchChar) { @@ -75,13 +89,6 @@ interface RichCharSequence extends CharSequence { return StringUtils.startsWith(this, prefix); } - - /** @see CharSequenceUtils#subSequence(CharSequence, int) */ - default CharSequence subSequence(int start) { - return subSequence(start, length()); - } - - /** @see Pattern#matches(String, CharSequence) */ default boolean matches(String regex) { return Pattern.matches(regex, this); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/SharingCharSeq.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SharingCharSeq.java similarity index 93% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/SharingCharSeq.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SharingCharSeq.java index bb6be1b0b4..37aa8e5509 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/SharingCharSeq.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SharingCharSeq.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.internal; +package net.sourceforge.pmd.lang.ast.impl; import java.util.Objects; @@ -53,7 +53,7 @@ public final class SharingCharSeq implements RichCharSequence { } @Override - public SharingCharSeq subSequence(int start, int end) { + public RichCharSequence subSequence(int start, int end) { if (start < 0 || end > count || start > end) { throw new IndexOutOfBoundsException("Invalid range: [" + start + "," + end + "[ not in [0," + count + "["); } @@ -61,7 +61,7 @@ public final class SharingCharSeq implements RichCharSequence { } @Override - public SharingCharSeq subSequence(int start) { + public RichCharSequence subSequence(int start) { return subSequence(start, count); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/SingletonNodeStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SingletonNodeStream.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/SingletonNodeStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/SingletonNodeStream.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/StreamImpl.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/StreamImpl.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/StreamImpl.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/StreamImpl.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/TraversalUtils.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TraversalUtils.java similarity index 100% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/internal/TraversalUtils.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TraversalUtils.java diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java index 554e9194c3..5c33e8b9be 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java @@ -14,6 +14,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.lang.ast.RootNode; +import net.sourceforge.pmd.lang.ast.impl.RichCharSequence; import net.sourceforge.pmd.lang.java.typeresolution.ClassTypeResolver; // FUTURE Change this class to extend from SimpleJavaNode, as TypeNode is not appropriate (unless I'm wrong) @@ -22,7 +23,7 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ro private ClassTypeResolver classTypeResolver; private List comments; private Map noPmdComments = Collections.emptyMap(); - private CharSequence fileText; + private RichCharSequence fileText; ASTCompilationUnit(int id) { super(id); @@ -41,12 +42,12 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ro } @Override - public CharSequence getText() { + public RichCharSequence getText() { return fileText; } - void setFileText(CharSequence fileText) { + void setFileText(RichCharSequence fileText) { this.fileText = fileText; } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/AbstractJavaNode.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/AbstractJavaNode.java index 20bf26230e..e13a00f003 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/AbstractJavaNode.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/AbstractJavaNode.java @@ -10,6 +10,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.lang.ast.AbstractNode; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.Node; +import net.sourceforge.pmd.lang.ast.impl.RichCharSequence; import net.sourceforge.pmd.lang.java.symbols.table.JSymbolTable; import net.sourceforge.pmd.lang.symboltable.Scope; @@ -20,7 +21,7 @@ abstract class AbstractJavaNode extends AbstractNode implements JavaNode { private JSymbolTable symbolTable; private Comment comment; private ASTCompilationUnit root; - private CharSequence text; + private RichCharSequence text; AbstractJavaNode(int id) { super(id); @@ -107,7 +108,7 @@ abstract class AbstractJavaNode extends AbstractNode implements JavaNode { @Override - public CharSequence getText() { + public RichCharSequence getText() { if (text == null) { text = getRoot().getText().subSequence(getStartOffset(), getEndOffset()); } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaNode.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaNode.java index 90fc8d6828..74b4c2e527 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaNode.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaNode.java @@ -9,6 +9,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.ast.impl.RichCharSequence; import net.sourceforge.pmd.lang.java.symbols.table.JSymbolTable; import net.sourceforge.pmd.lang.symboltable.ScopedNode; @@ -85,10 +86,6 @@ public interface JavaNode extends ScopedNode { @Override JavaNode jjtGetParent(); - - ASTCompilationUnit getRoot(); - - GenericToken jjtGetFirstToken(); @@ -119,7 +116,7 @@ public interface JavaNode extends ScopedNode { int getEndOffset(); - CharSequence getText(); + RichCharSequence getText(); /**