From ad7e9eb75d8bc261672012e4d35196bddbae1750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 11 Jan 2020 00:23:34 +0100 Subject: [PATCH] Make token document store first token --- javacc-wrapper.xml | 2 +- .../pmd/lang/ast/impl/TokenDocument.java | 1 - .../ast/impl/javacc/AbstractJjtreeNode.java | 14 +- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 17 +- .../ast/impl/javacc/JavaccTokenDocument.java | 75 +++++- pmd-java/etc/grammar/Java.jjt | 4 +- pmd-java/src/main/ant/alljavacc.xml | 233 ------------------ .../pmd/lang/java/ast/InternalApiBridge.java | 2 +- ...kenFactory.java => JavaTokenDocument.java} | 35 ++- .../java/internal/JavaLanguageParser.java | 2 +- .../pmd/lang/java/ast/KotlinTestingDsl.kt | 1 + 11 files changed, 118 insertions(+), 268 deletions(-) delete mode 100644 pmd-java/src/main/ant/alljavacc.xml rename pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/{JavaTokenFactory.java => JavaTokenDocument.java} (69%) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index bc0b1f9106..295c0ec152 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -202,7 +202,7 @@ - + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TokenDocument.java index 7129675e14..26689f91e5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TokenDocument.java @@ -28,7 +28,6 @@ public abstract class TokenDocument { return fullText; } - public int lineNumberFromOffset(int offset) { return positioner.lineNumberFromOffset(offset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java index 6b5e59e1bc..8b09d84a52 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java @@ -29,11 +29,21 @@ public abstract class AbstractJjtreeNode extends AbstractNode im @Override public CharSequence getText() { - String fullText = ((JavaccToken) jjtGetFirstToken()).document.getFullText(); - int realEnd = min(getEndOffset(), fullText.length()); // EOF token messes things up? + String fullText = jjtGetFirstToken().document.getFullText(); + int realEnd = min(getEndOffset(), fullText.length()); // TODO EOF token messes things up? return fullText.substring(getStartOffset(), realEnd); } + @Override + public JavaccToken jjtGetFirstToken() { + return (JavaccToken) super.jjtGetFirstToken(); + } + + @Override + public JavaccToken jjtGetLastToken() { + return (JavaccToken) super.jjtGetLastToken(); + } + @Override public N jjtGetChild(int index) { return (N) super.jjtGetChild(index); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 4067f133c3..5ed61ed6bc 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import org.checkerframework.checker.nullness.qual.NonNull; + import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.GenericToken; @@ -216,11 +218,7 @@ public class JavaccToken implements GenericToken { */ public static JavaccToken implicitBefore(JavaccToken next) { - JavaccToken implicit = new JavaccToken(IMPLICIT_TOKEN, - "", - next.getStartInDocument(), - next.getStartInDocument(), - next.document); + JavaccToken implicit = newImplicit(next.getStartInDocument(), next.document); // insert it right before the next token // as a special token @@ -236,5 +234,14 @@ public class JavaccToken implements GenericToken { return implicit; } + @NonNull + public static JavaccToken newImplicit(int offset, JavaccTokenDocument document) { + return new JavaccToken(IMPLICIT_TOKEN, + "", + offset, + offset, + document); + } + } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 4d96ea72be..34760e2d32 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -4,29 +4,100 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; /** - * Token document for Javacc implementations. + * Token document for Javacc implementations. This is a helper object + * for generated token managers. */ public class JavaccTokenDocument extends TokenDocument { + private JavaccToken first; public JavaccTokenDocument(String fullText) { super(fullText); } + /** + * Open the document. This is only meant to be used by + * Javacc-generated file. + * + * @throws IllegalStateException If the document has already been opened + */ + public JavaccToken open() { + synchronized (this) { + if (first != null) { + throw new RuntimeException("Document is already opened"); + } + first = JavaccToken.newImplicit(0, this); + } + return first; + } - protected String describeKind(int kind) { + /** + * Returns the first token of the token chain. + */ + // technically the first non-implicit, though this is stupid + public JavaccToken getFirstToken() { + if (first == null || first.next == null) { + throw new IllegalStateException("Document has not been opened"); + } + return first.next; + } + + /** + * Returns a string that describes the token kind. + * + * @param kind Kind of token + * + * @return A descriptive string + */ + public final @NonNull String describeKind(int kind) { if (kind == JavaccToken.IMPLICIT_TOKEN) { return "implicit token"; } + String impl = describeKindImpl(kind); + if (impl != null) { + return impl; + } return "token of kind " + kind; } + + /** + * Describe the given kind. If this returns a non-null value, then + * that's what {@link #describeKind(int)} will use. Otherwise a default + * implementation is used. + * + *

An implementation typically uses the JavaCC-generated array + * named {@code Constants.tokenImage}. Remember to + * check the bounds of the array. + * + * @param kind Kind of token + * + * @return A descriptive string, or null to use default + */ + protected @Nullable String describeKindImpl(int kind) { + return null; + } + + /** + * Creates a new token with the given kind. This is called back to + * by JavaCC-generated token managers (jjFillToken). + * + * @param kind Kind of the token + * @param cs Char stream of the file. This can be used to get text + * coordinates and the image + * @param image Shared instance of the image token. If this is non-null, + * then no call to {@link CharStream#GetImage()} should be + * issued. + * + * @return A new token + */ public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { return new JavaccToken( kind, diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index 0b383ad619..c7f73ce97e 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -2757,14 +2757,14 @@ void AssertStatement() : void RUNSIGNEDSHIFT(): // TODO 7.0.0 make #void {} { - LOOKAHEAD({ JavaTokenFactory.getRealKind(getToken(1)) == RUNSIGNEDSHIFT}) + LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RUNSIGNEDSHIFT}) ">" ">" ">" } void RSIGNEDSHIFT(): // TODO 7.0.0 make #void {} { - LOOKAHEAD({ JavaTokenFactory.getRealKind(getToken(1)) == RSIGNEDSHIFT}) + LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RSIGNEDSHIFT}) ">" ">" } diff --git a/pmd-java/src/main/ant/alljavacc.xml b/pmd-java/src/main/ant/alljavacc.xml deleted file mode 100644 index 744ef4f31c..0000000000 --- a/pmd-java/src/main/ant/alljavacc.xml +++ /dev/null @@ -1,233 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Note that when the total number of children is definitely known, you - * can use "definite nodes", ie write the expected number of children (including - * the ones to the left) in the JJTree annotation (eg {@code #AdditiveExpression(2)}). - * So this is only useful when the number of children of the current node is not certain. - * - *

This method does not affect the stack unless the current jjtThis is - * closed in the future. - */ - public void extendLeft() { - mk--; - } - - /** - * Peek the nth node from the top of the stack. - * peekNode(0) == peekNode() - */ - public Node peekNode(int n) { - return nodes.get(nodes.size() - n - 1); - } - - public boolean isInjectionPending() { - return numPendingInjection > 0; - } - - /** If non-zero, then the top "n" nodes of the stack will be injected as the first children of the next - * node to be opened. This is not very flexible, but it's enough. The grammar needs to take - * care of the order in which nodes are opened in a few places, in most cases this just means using - * eg A() B() #N(2) instead of (A() B()) #N, so as not to open N before A. - */ - private int numPendingInjection; - - public void injectRight(int n) { - numPendingInjection = n; - } - - /* Pushes a node on to the stack. */]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public class - - - - - - - - - - - diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java index ac6e056d9e..a9d90fa14b 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java @@ -29,7 +29,7 @@ public final class InternalApiBridge { } public static ASTCompilationUnit parseInternal(String fileName, Reader source, int jdkVersion, boolean preview, ParserOptions options) { - JavaParser parser = new JavaParser(CharStreamFactory.javaCharStream(source)); + JavaParser parser = new JavaParser(CharStreamFactory.javaCharStream(source, JavaTokenDocument::new)); String suppressMarker = options.getSuppressMarker(); if (suppressMarker != null) { parser.setSuppressMarker(suppressMarker); diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java similarity index 69% rename from pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java rename to pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 84a7da3bc3..e7df016bbc 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -4,24 +4,30 @@ package net.sourceforge.pmd.lang.java.ast; +import org.checkerframework.checker.nullness.qual.Nullable; + import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaCharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; /** - * Support methods for the token manager. The call to {@link #newToken(int, CharStream)} - * is hacked in via search/replace on {@link JavaParserTokenManager}. + * {@link JavaccTokenDocument} for Java. */ -final class JavaTokenFactory { - - private JavaTokenFactory() { +final class JavaTokenDocument extends JavaccTokenDocument { + JavaTokenDocument(String fullText) { + super(fullText); } - static JavaccToken newToken(int kind, CharStream charStream) { - JavaCharStream jcs = (JavaCharStream) charStream; + @Override + protected @Nullable String describeKindImpl(int kind) { + return 0 <= kind && kind < JavaParserConstants.tokenImage.length + ? JavaParserConstants.tokenImage[kind] + : null; + } + @Override + public JavaccToken createToken(int kind, CharStream jcs, @Nullable String image) { switch (kind) { case JavaParserConstants.RUNSIGNEDSHIFT: case JavaParserConstants.RSIGNEDSHIFT: @@ -48,18 +54,7 @@ final class JavaTokenFactory { ); default: - // Most tokens have an entry in there, it's used to share the - // image string for keywords & punctuation. Those represent ~40% - // of token instances - String image = JavaParserTokenManager.jjstrLiteralImages[kind]; - - return new JavaccToken( - kind, - image == null ? charStream.GetImage() : image, - jcs.getStartOffset(), - jcs.getEndOffset(), - jcs.getTokenDocument() - ); + return super.createToken(kind, jcs, image); } } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/internal/JavaLanguageParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/internal/JavaLanguageParser.java index 41b84e2011..3ca8be6aee 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/internal/JavaLanguageParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/internal/JavaLanguageParser.java @@ -11,9 +11,9 @@ import net.sourceforge.pmd.lang.AbstractParser; import net.sourceforge.pmd.lang.ParserOptions; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.Node; +import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.java.JavaTokenManager; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; -import net.sourceforge.pmd.lang.java.ast.ParseException; /** * Adapter for the JavaParser, using the specified grammar version. diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt index 71756d18f8..73721dc325 100644 --- a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt @@ -3,6 +3,7 @@ package net.sourceforge.pmd.lang.java.ast import io.kotlintest.matchers.string.shouldContain import io.kotlintest.shouldThrow import net.sourceforge.pmd.lang.ast.Node +import net.sourceforge.pmd.lang.ast.ParseException import net.sourceforge.pmd.lang.ast.test.* import net.sourceforge.pmd.lang.java.JavaParsingHelper