phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make token document store first token

Browse Source
This commit is contained in:
Clément Fournier
2020-01-11 00:23:34 +01:00
parent f715e6af72
commit ad7e9eb75d
11 changed files with 118 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
javacc-wrapper.xml
View File

@ -202,7 +202,7 @@
</replaceregexp>
<replace token="new Token()" value="${ast-impl-package}.JavaccToken.undefined()">
<replace token="new Token()" value="token_source.input_stream.getTokenDocument().open()">
<fileset dir="${target-package-dir}" />
</replace>

1
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/TokenDocument.java
View File

@ -28,7 +28,6 @@ public abstract class TokenDocument<T extends GenericToken> {
return fullText;
}
public int lineNumberFromOffset(int offset) {
return positioner.lineNumberFromOffset(offset);
}

14
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java
View File

@ -29,11 +29,21 @@ public abstract class AbstractJjtreeNode<N extends Node> extends AbstractNode im
@Override
public CharSequence getText() {
String fullText = ((JavaccToken) jjtGetFirstToken()).document.getFullText();
int realEnd = min(getEndOffset(), fullText.length()); // EOF token messes things up?
String fullText = jjtGetFirstToken().document.getFullText();
int realEnd = min(getEndOffset(), fullText.length()); // TODO EOF token messes things up?
return fullText.substring(getStartOffset(), realEnd);
}
@Override
public JavaccToken jjtGetFirstToken() {
return (JavaccToken) super.jjtGetFirstToken();
}
@Override
public JavaccToken jjtGetLastToken() {
return (JavaccToken) super.jjtGetLastToken();
}
@Override
public N jjtGetChild(int index) {
return (N) super.jjtGetChild(index);

17
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java
View File

@ -4,6 +4,8 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import org.checkerframework.checker.nullness.qual.NonNull;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.GenericToken;
@ -216,11 +218,7 @@ public class JavaccToken implements GenericToken {
*/
public static JavaccToken implicitBefore(JavaccToken next) {
JavaccToken implicit = new JavaccToken(IMPLICIT_TOKEN,
"",
next.getStartInDocument(),
next.getStartInDocument(),
next.document);
JavaccToken implicit = newImplicit(next.getStartInDocument(), next.document);
// insert it right before the next token
// as a special token
@ -236,5 +234,14 @@ public class JavaccToken implements GenericToken {
return implicit;
}
@NonNull
public static JavaccToken newImplicit(int offset, JavaccTokenDocument document) {
return new JavaccToken(IMPLICIT_TOKEN,
"",
offset,
offset,
document);
}
}

75
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java
View File

@ -4,29 +4,100 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
/**
* Token document for Javacc implementations.
* Token document for Javacc implementations. This is a helper object
* for generated token managers.
*/
public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
private JavaccToken first;
public JavaccTokenDocument(String fullText) {
super(fullText);
}
/**
* Open the document. This is only meant to be used by
* Javacc-generated file.
*
* @throws IllegalStateException If the document has already been opened
*/
public JavaccToken open() {
synchronized (this) {
if (first != null) {
throw new RuntimeException("Document is already opened");
}
first = JavaccToken.newImplicit(0, this);
}
return first;
}
protected String describeKind(int kind) {
/**
* Returns the first token of the token chain.
*/
// technically the first non-implicit, though this is stupid
public JavaccToken getFirstToken() {
if (first == null || first.next == null) {
throw new IllegalStateException("Document has not been opened");
}
return first.next;
}
/**
* Returns a string that describes the token kind.
*
* @param kind Kind of token
*
* @return A descriptive string
*/
public final @NonNull String describeKind(int kind) {
if (kind == JavaccToken.IMPLICIT_TOKEN) {
return "implicit token";
}
String impl = describeKindImpl(kind);
if (impl != null) {
return impl;
}
return "token of kind " + kind;
}
/**
* Describe the given kind. If this returns a non-null value, then
* that's what {@link #describeKind(int)} will use. Otherwise a default
* implementation is used.
*
* <p>An implementation typically uses the JavaCC-generated array
* named {@code <parser name>Constants.tokenImage}. Remember to
* check the bounds of the array.
*
* @param kind Kind of token
*
* @return A descriptive string, or null to use default
*/
protected @Nullable String describeKindImpl(int kind) {
return null;
}
/**
* Creates a new token with the given kind. This is called back to
* by JavaCC-generated token managers (jjFillToken).
*
* @param kind Kind of the token
* @param cs Char stream of the file. This can be used to get text
* coordinates and the image
* @param image Shared instance of the image token. If this is non-null,
* then no call to {@link CharStream#GetImage()} should be
* issued.
*
* @return A new token
*/
public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) {
return new JavaccToken(
kind,

4
pmd-java/etc/grammar/Java.jjt
View File

@ -2757,14 +2757,14 @@ void AssertStatement() :
void RUNSIGNEDSHIFT(): // TODO 7.0.0 make #void
{}
{
LOOKAHEAD({ JavaTokenFactory.getRealKind(getToken(1)) == RUNSIGNEDSHIFT})
LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RUNSIGNEDSHIFT})
">" ">" ">"
}
void RSIGNEDSHIFT(): // TODO 7.0.0 make #void
{}
{
LOOKAHEAD({ JavaTokenFactory.getRealKind(getToken(1)) == RSIGNEDSHIFT})
LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RSIGNEDSHIFT})
">" ">"
}

233
pmd-java/src/main/ant/alljavacc.xml
View File

@ -1,233 +0,0 @@
<project name="pmd" default="alljavacc" basedir="../../">
<property name="javacc-home.path" value="target/lib" />
<property name="target-package-dir" value="${target}/net/sourceforge/pmd/lang/java/ast" />
<property name="stamp-file" value="${target}/../../last-generated-timestamp"/>
<!-- Matches the names of deprecated node types to add a @Deprecated annotation -->
<property name="deprecated-nodes-pattern" value="ASTR(UN)?SIGNEDSHIFT" />
<!-- Visitor names -->
<property name="base-visitor-interface-name" value="JavaParserVisitor" />
<property name="base-visitor-interface-file" value="${target-package-dir}/${base-visitor-interface-name}.java" />
<property name="generic-sideeffect-visitor-interface-name" value="SideEffectingVisitor" />
<property name="generic-sideeffect-visitor-interface-file"
value="${target-package-dir}/${generic-sideeffect-visitor-interface-name}.java" />
<property name="ast-core-package" value="net.sourceforge.pmd.lang.ast" />
<property name="ast-impl-package" value="${ast-core-package}.impl.javacc" />
<!-- TARGETS -->
<target name="alljavacc"
description="Generates all JavaCC aspects within PMD"
depends="checkUpToDate,init,javajjtree,cleanup" />
<target name="checkUpToDate">
<uptodate property="javaccBuildNotRequired" targetfile="${stamp-file}">
<srcfiles dir="etc/grammar" includes="*.jj*"/>
<srcfiles file="src/main/ant/alljavacc.xml" />
</uptodate>
<echo message="up to date check: javaccBuildNotRequired=${javaccBuildNotRequired}"/>
</target>
<target name="init" unless="javaccBuildNotRequired">
<mkdir dir="${javacc-home.path}" />
<copy file="${javacc.jar}" tofile="${javacc-home.path}/javacc.jar" />
<mkdir dir="${target}"/>
<touch file="${stamp-file}"/>
</target>
<target name="cleanup">
<delete dir="${javacc-home.path}" />
</target>
<target name="javajjtree" description="Generates the Java parser and AST source files" unless="javaccBuildNotRequired">
<delete dir="${target-package-dir}" />
<mkdir dir="${target-package-dir}" />
<jjtree target="etc/grammar/Java.jjt"
outputdirectory="${target-package-dir}"
javacchome="${javacc-home.path}" />
<!-- Ensure generated using CharStream interface -->
<javacc static="false"
usercharstream="true"
target="${target-package-dir}/Java.jj"
outputdirectory="${target-package-dir}"
javacchome="${javacc-home.path}" />
<delete file="${target-package-dir}/Node.java" />
<delete file="${target-package-dir}/SimpleNode.java" />
<delete file="${target-package-dir}/CharStream.java" />
<delete file="${target-package-dir}/TokenMgrError.java" />
<replace file="${target-package-dir}/JJTJavaParserState.java">
<replacefilter token="/*" value="/**"/>
<replacetoken><![CDATA[ /* Pushes a node on to the stack. */]]></replacetoken>
<replacevalue>
<![CDATA[
/**
* Extend the number of children of the current node of one to the left.
* If the node is closed, one additional node from the stack will be popped
* and added to its children. This allows mimicking "left-recursive" nodes,
* while keeping the parsing iterative.
*
* <p>Note that when the total number of children is definitely known, you
* can use "definite nodes", ie write the expected number of children (including
* the ones to the left) in the JJTree annotation (eg {@code #AdditiveExpression(2)}).
* So this is only useful when the number of children of the current node is not certain.
*
* <p>This method does not affect the stack unless the current jjtThis is
* closed in the future.
*/
public void extendLeft() {
mk--;
}
/**
* Peek the nth node from the top of the stack.
* peekNode(0) == peekNode()
*/
public Node peekNode(int n) {
return nodes.get(nodes.size() - n - 1);
}
public boolean isInjectionPending() {
return numPendingInjection > 0;
}
/** If non-zero, then the top "n" nodes of the stack will be injected as the first children of the next
* node to be opened. This is not very flexible, but it's enough. The grammar needs to take
* care of the order in which nodes are opened in a few places, in most cases this just means using
* eg A() B() #N(2) instead of (A() B()) #N, so as not to open N before A.
*/
private int numPendingInjection;
public void injectRight(int n) {
numPendingInjection = n;
}
/* Pushes a node on to the stack. */]]>
</replacevalue>
</replace>
<replace file="${target-package-dir}/JJTJavaParserState.java">
<!-- This is in openNodeScope. -->
<!-- If injection is pending, we bump the arity of the opened node. -->
<!-- When it's closed, it will enclose the injected node. -->
<replacetoken><![CDATA[mk = sp;]]></replacetoken>
<replacevalue><![CDATA[
mk = sp;
if (isInjectionPending()) {
mk -= numPendingInjection;
numPendingInjection = 0;
}]]>
</replacevalue>
</replace>
<replace token="new Token()" value="${ast-impl-package}.JavaccToken.undefined()">
<fileset dir="${target-package-dir}" />
</replace>
<!-- Map Javacc names to our names -->
<replaceregexp flags="g">
<regexp pattern="\bToken\b" />
<substitution expression="${ast-impl-package}.JavaccToken" />
<fileset dir="${target-package-dir}" />
</replaceregexp>
<replace file="${target-package-dir}/JavaParserTokenManager.java">
<replacetoken><![CDATA[t = JavaTokenFactory.newToken(jjmatchedKind, curTokenImage);
t.beginLine = beginLine;
t.endLine = endLine;
t.beginColumn = beginColumn;
t.endColumn = endColumn;]]></replacetoken>
<replacevalue>
<![CDATA[t = JavaTokenFactory.newToken(jjmatchedKind, curTokenImage, beginLine, endLine, beginColumn, endColumn);]]></replacevalue>
</replace>
<replaceregexp file="${target-package-dir}/JavaParserTokenManager.java" flags="s">
<regexp pattern="protected net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken jjFillToken.*?}" />
<substitution
expression="protected net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken jjFillToken() {return JavaTokenFactory.newToken(jjmatchedKind, input_stream);}"/>
</replaceregexp>
<replace token=".image" value=".getImage()">
<fileset dir="${target-package-dir}"/>
</replace>
<replace token=".beginLine" value=".getBeginLine()">
<fileset dir="${target-package-dir}"/>
</replace>
<replace token=".beginColumn" value=".getBeginColumn()">
<fileset dir="${target-package-dir}"/>
</replace>
<replace file="${target-package-dir}/JavaParserTokenManager.java"
token="public class JavaParserTokenManager"
value="import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; public class JavaParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />
<replace file="${target-package-dir}/JavaParser.java"
token="throw new Error"
value="throw new RuntimeException" />
<replace file="${target-package-dir}/ParseException.java"
token="extends Exception"
value="extends net.sourceforge.pmd.lang.ast.ParseException" />
<!-- VISITORS -->
<!-- Base visitor with Object everywhere -->
<!-- We perform most changes like adding default methods, etc on this one -->
<!-- Changes are then copied on other visitors -->
<replace file="${base-visitor-interface-file}">
<replacefilter token="JavaParserVisitor" value="${base-visitor-interface-name}" />
<replacefilter token="SimpleNode" value="JavaNode" />
<!-- Default methods -->
<replacefilter token="public Object visit(" value="default Object visit(" />
<replacefilter token=");" value=") { return visit((JavaNode) node, data); }" />
<replacefilter token="default Object visit(JavaNode node, Object data) { return visit((JavaNode) node, data); }"
value="default Object visit(JavaNode node, Object data) { return node.childrenAccept(this, data); }" />
</replace>
<!-- Deprecated nodes -->
<replaceregexp file="${base-visitor-interface-file}"
match="default Object visit\((${deprecated-nodes-pattern})"
byline="true"
replace="@Deprecated \0" />
<!-- Side effecting visitor, no return type, one generic parameter -->
<copy file="${base-visitor-interface-file}" tofile="${generic-sideeffect-visitor-interface-file}" />
<replace file="${generic-sideeffect-visitor-interface-file}">
<replacefilter token="${base-visitor-interface-name}" value="${generic-sideeffect-visitor-interface-name}&lt;T>" />
<replacefilter token="Object" value="T" />
<replacefilter token="T visit" value="void visit" />
<replacefilter token="return data;" value="" />
<replacefilter token="return " value="" />
</replace>
<replace file="${target-package-dir}/JJTJavaParserState.java">
<replacetoken>public class</replacetoken>
<replacevalue><![CDATA[import net.sourceforge.pmd.lang.ast.Node;
public class]]></replacevalue>
</replace>
<delete>
<fileset dir="${target-package-dir}">
<include name="AST*.java" />
<include name="Token.java"/>
</fileset>
</delete>
</target>
</project>

2
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java
View File

@ -29,7 +29,7 @@ public final class InternalApiBridge {
}
public static ASTCompilationUnit parseInternal(String fileName, Reader source, int jdkVersion, boolean preview, ParserOptions options) {
JavaParser parser = new JavaParser(CharStreamFactory.javaCharStream(source));
JavaParser parser = new JavaParser(CharStreamFactory.javaCharStream(source, JavaTokenDocument::new));
String suppressMarker = options.getSuppressMarker();
if (suppressMarker != null) {
parser.setSuppressMarker(suppressMarker);

35
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java → pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java
View File

@ -4,24 +4,30 @@
package net.sourceforge.pmd.lang.java.ast;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaCharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
/**
* Support methods for the token manager. The call to {@link #newToken(int, CharStream)}
* is hacked in via search/replace on {@link JavaParserTokenManager}.
* {@link JavaccTokenDocument} for Java.
*/
final class JavaTokenFactory {
private JavaTokenFactory() {
final class JavaTokenDocument extends JavaccTokenDocument {
JavaTokenDocument(String fullText) {
super(fullText);
}
static JavaccToken newToken(int kind, CharStream charStream) {
JavaCharStream jcs = (JavaCharStream) charStream;
@Override
protected @Nullable String describeKindImpl(int kind) {
return 0 <= kind && kind < JavaParserConstants.tokenImage.length
? JavaParserConstants.tokenImage[kind]
: null;
}
@Override
public JavaccToken createToken(int kind, CharStream jcs, @Nullable String image) {
switch (kind) {
case JavaParserConstants.RUNSIGNEDSHIFT:
case JavaParserConstants.RSIGNEDSHIFT:
@ -48,18 +54,7 @@ final class JavaTokenFactory {
);
default:
// Most tokens have an entry in there, it's used to share the
// image string for keywords & punctuation. Those represent ~40%
// of token instances
String image = JavaParserTokenManager.jjstrLiteralImages[kind];
return new JavaccToken(
kind,
image == null ? charStream.GetImage() : image,
jcs.getStartOffset(),
jcs.getEndOffset(),
jcs.getTokenDocument()
);
return super.createToken(kind, jcs, image);
}
}

2
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/internal/JavaLanguageParser.java
View File

@ -11,9 +11,9 @@ import net.sourceforge.pmd.lang.AbstractParser;
import net.sourceforge.pmd.lang.ParserOptions;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.java.JavaTokenManager;
import net.sourceforge.pmd.lang.java.ast.InternalApiBridge;
import net.sourceforge.pmd.lang.java.ast.ParseException;
/**
* Adapter for the JavaParser, using the specified grammar version.

1
pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt
View File

@ -3,6 +3,7 @@ package net.sourceforge.pmd.lang.java.ast
import io.kotlintest.matchers.string.shouldContain
import io.kotlintest.shouldThrow
import net.sourceforge.pmd.lang.ast.Node
import net.sourceforge.pmd.lang.ast.ParseException
import net.sourceforge.pmd.lang.ast.test.*
import net.sourceforge.pmd.lang.java.JavaParsingHelper