hiifong/pmd
1
0 Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Merge pull request #4044 from oowekyala:text-utils-javacc

Browse Source 
[core] Text documents escapes #4044
This commit is contained in:
Andreas Dangel committed 2022-09-09 16:19:47 +02:00
2 parents 42523825d5 f3dbdb700c
commit eec3959e75
92 files changed
+1918 -1016
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

javacc-wrapper.xml
+33 -3
View File
@@ -59,8 +59,8 @@
<property name="base-class-name" value="Abstract${lang-name}Node" />
<!-- This will be moved to impl package when all language modules have been ported -->
<property name="base-tokenmgr" value="${ast-api-package}.AbstractTokenManager"/>
<property name="charstream-itf" value="${ast-api-package}.CharStream"/>
<property name="base-tokenmgr" value="${ast-impl-package}.AbstractTokenManager"/>
<property name="charstream-itf" value="${ast-impl-package}.CharStream"/>
<property name="tokenmgr-name" value="${parser-name}TokenManager" />
@@ -284,6 +284,25 @@
<substitution expression="protected Token jjFillToken() {return input_stream.getTokenDocument().createToken(jjmatchedKind, input_stream, jjstrLiteralImages[jjmatchedKind]);}" />
</replaceregexp>
<!-- Renamed CharStream methods -->
<replaceregexp flags="g">
<file name="${tokenmgr-file}" />
<regexp pattern="image.append\(input_stream.GetSuffix\(" />
<substitution expression="input_stream.appendSuffix(image,(" />
</replaceregexp>
<replaceregexp flags="g">
<file name="${tokenmgr-file}" />
<regexp pattern="input_stream.GetImage" />
<substitution expression="input_stream.getTokenImage" />
</replaceregexp>
<replaceregexp flags="g">
<file name="${tokenmgr-file}" />
<regexp pattern="input_stream.BeginToken" />
<substitution expression="input_stream.markTokenStart" />
</replaceregexp>
<!-- This is used to allow for tokens to be immutable. The lexical actions
return the new token instead of mutating it. -->
<replaceregexp flags="sg">
@@ -396,7 +415,7 @@ public final class ${token-constants-name} \{${line.separator}
* be used as a basis for a CPD Tokenizer.
*/
@net.sourceforge.pmd.annotation.InternalApi
public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.CharStream cs) {
public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) {
return new %%%TOKEN_MGR_NAME%%%(cs);
}
@@ -405,6 +424,17 @@ public final class ${token-constants-name} \{${line.separator}
<fileset file="${token-constants-file}" />
</replace>
<replace>
<replacetoken>};</replacetoken>
<replacevalue><![CDATA[
};
/** Nams of the tokens, each index corresponds to a kind. See also {@link #describe(int)}. */
public static final java.util.List<String> TOKEN_NAMES = java.util.Collections.unmodifiableList(java.util.Arrays.asList(tokenImage));
]]> </replacevalue>
<fileset file="${token-constants-file}" />
</replace>
<replaceregexp>
<regexp pattern="%%%TOKEN_MGR_NAME%%%" />
<substitution expression="${tokenmgr-name}" />
pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java
+2 -4
View File
@@ -37,9 +37,7 @@ public final class ApexParser implements Parser {
final Compilation astRoot = CompilerService.INSTANCE.parseApex(task.getTextDocument());
if (astRoot == null) {
throw new ParseException("Couldn't parse the source - there is not root node - Syntax Error??");
}
assert astRoot != null : "Normally replaced by Compilation.INVALID";
String property = task.getProperties().getProperty(MULTIFILE_DIRECTORY);
ApexMultifileAnalysis analysisHandler = ApexMultifileAnalysis.getAnalysisInstance(property);
@@ -48,7 +46,7 @@ public final class ApexParser implements Parser {
final ApexTreeBuilder treeBuilder = new ApexTreeBuilder(task);
return treeBuilder.buildTree(astRoot, analysisHandler);
} catch (apex.jorje.services.exception.ParseException e) {
throw new ParseException(e);
throw new ParseException(e).setFileName(task.getFileDisplayName());
}
}
}
pmd-core/pom.xml
-23
View File
@@ -13,29 +13,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<inherited>true</inherited>
<executions>
<execution>
<id>generate-sources</id>
<phase>generate-sources</phase>
<configuration>
<target>
<ant antfile="src/main/ant/alljavacc.xml">
<property name="target" value="${project.build.directory}/generated-sources/javacc" />
<property name="javacc.jar" value="${javacc.jar}" />
</ant>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
pmd-core/src/main/ant/alljavacc.xml
-130
View File
@@ -1,130 +0,0 @@
<project name="pmd" default="alljavacc" basedir="../../">
<property name="javacc-home.path" value="target/lib" />
<property name="tmp-package" value="net.sourceforge.pmd.lang.ast.dummy" />
<property name="tmp-package.dir" value="${target}/net/sourceforge/pmd/lang/ast/dummy" />
<property name="base-ast-package" value="net.sourceforge.pmd.lang.ast" />
<property name="base-ast-package.dir" value="${target}/net/sourceforge/pmd/lang/ast" />
<property name="target-package" value="${base-ast-package}.impl.javacc" />
<property name="target-package.dir" value="${base-ast-package.dir}/impl/javacc" />
<target name="alljavacc"
description="Generates all JavaCC aspects within PMD"
depends="checkUpToDate,init,dummyjjtree,cleanup" />
<target name="checkUpToDate">
<uptodate property="javaccBuildNotRequired" targetfile="${target}/last-generated-timestamp">
<srcfiles dir="etc/grammar" includes="*.jj*"/>
</uptodate>
<echo message="up to date check: javaccBuildNotRequired=${javaccBuildNotRequired}"/>
</target>
<target name="init" unless="javaccBuildNotRequired">
<mkdir dir="${javacc-home.path}" />
<copy file="${javacc.jar}" tofile="${javacc-home.path}/javacc.jar" />
<mkdir dir="${target}"/>
<touch file="${target}/last-generated-timestamp"/>
</target>
<target name="cleanup">
<delete dir="${javacc-home.path}" />
</target>
<target name="dummyjjtree" description="Generates the reusable JavaCC aspects" unless="javaccBuildNotRequired">
<delete dir="${tmp-package.dir}" />
<mkdir dir="${tmp-package.dir}" />
<echo>Using JavaCC home: ${javacc-home.path}</echo>
<java fork="true"
classname="jjtree"
classpath="${javacc-home.path}/javacc.jar">
<sysproperty key="file.encoding" value="UTF-8" />
<arg value="-OUTPUT_DIRECTORY:${tmp-package.dir}" />
<arg value="etc/grammar/dummy.jjt" />
</java>
<!-- Generate CharStream interface -->
<java fork="true"
classname="javacc"
classpath="${javacc-home.path}/javacc.jar">
<sysproperty key="file.encoding" value="UTF-8" />
<arg value="-USER_CHAR_STREAM:true" />
<arg value="-OUTPUT_DIRECTORY:${tmp-package.dir}" />
<arg value="${tmp-package.dir}/dummy.jj" />
</java>
<replace file="${tmp-package.dir}/CharStream.java"
token="interface"
value="@Deprecated @net.sourceforge.pmd.annotation.InternalApi interface" />
<!-- Generate ASCII w/ Unicode Escapes CharStream implementation -->
<javacc usercharstream="false"
unicodeinput="true"
javaunicodeescape="true"
static="false"
target="${tmp-package.dir}/dummy.jj"
outputdirectory="${tmp-package.dir}"
javacchome="${javacc-home.path}" />
<replace file="${tmp-package.dir}/JavaCharStream.java"
token="${tmp-package}"
value="${target-package}">
<fileset dir="${tmp-package.dir}">
</fileset>
</replace>
<!-- Patch JavaCharStream -->
<antcall target="patch-char-stream">
<param name="cs.prefix" value="Java" />
</antcall>
<delete dir="${tmp-package.dir}" />
</target>
<target name="patch-char-stream">
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="${cs.prefix}CharStream"
value="${cs.prefix}CharStreamBase"/>
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="class ${cs.prefix}CharStreamBase"
value="abstract class ${cs.prefix}CharStreamBase implements ${base-ast-package}.CharStream" />
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="/** Read a character. */"
value="protected boolean doEscape() { return true; }" />
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="if ((buffer[bufpos] = c = ReadByte()) == '\\')"
value="if ((buffer[bufpos] = c = ReadByte()) == '\\' &amp;&amp; doEscape())" />
<!-- This is in ExpandBuf, the exception this may throw is OutOfMemoryError, which is stupidly erased by javacc -->
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="throw new Error(t.getMessage())"
value="throw t" />
<!-- Set tab size of JavaCC to 1 -->
<replace file="${tmp-package.dir}/${cs.prefix}CharStream.java"
token="int tabSize = 8;"
value="int tabSize = 1;" />
<replaceregexp file="${tmp-package.dir}/${cs.prefix}CharStream.java">
<regexp pattern='throw new Error\("Invalid escape character at line " \+ line \+\s+" column " \+ column \+ "."\);' />
<substitution expression='throw new ${base-ast-package}.TokenMgrError(line, column, null, "Invalid unicode escape", null);' />
</replaceregexp>
<move overwrite="true"
file="${tmp-package.dir}/${cs.prefix}CharStream.java"
tofile="${target-package.dir}/${cs.prefix}CharStreamBase.java" />
</target>
</project>
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
+7 -7
View File
@@ -13,10 +13,10 @@ import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.document.CpdCompat;
import net.sourceforge.pmd.lang.document.TextDocument;
@@ -24,11 +24,11 @@ public abstract class JavaCCTokenizer implements Tokenizer {
@SuppressWarnings("PMD.CloseResource")
protected TokenManager<JavaccToken> getLexerForSource(TextDocument sourceCode) throws IOException {
return makeLexerImpl(makeCharStream(sourceCode));
return makeLexerImpl(CharStream.create(sourceCode, tokenBehavior()));
}
protected CharStream makeCharStream(TextDocument sourceCode) {
return CharStreamFactory.simpleCharStream(sourceCode);
protected TokenDocumentBehavior tokenBehavior() {
return TokenDocumentBehavior.DEFAULT;
}
protected abstract TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode);
@@ -55,7 +55,7 @@ public abstract class JavaCCTokenizer implements Tokenizer {
tokenEntries.add(processToken(tokenEntries, currentToken));
currentToken = tokenFilter.getNextToken();
}
} catch (TokenMgrError e) {
} catch (FileAnalysisException e) {
throw e.setFileName(sourceCode.getFileName());
} finally {
tokenEntries.add(TokenEntry.getEOF());
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java
-120
View File
@@ -1,120 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast;
import java.io.IOException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
/**
* PMD flavour of character streams used by JavaCC parsers.
*
* TODO for when all JavaCC languages are aligned:
* * rename methods to match decent naming conventions
* * move to impl.javacc package
*/
public interface CharStream {
/**
* Returns the next character from the input. After a {@link #backup(int)},
* some of the already read chars must be spit out again.
*
* @return The next character
*
* @throws IOException If the underlying char stream throws
*/
char readChar() throws IOException;
/**
* Calls {@link #readChar()} and returns its value, marking its position
* as the beginning of the next token. All characters must remain in
* the buffer between two successive calls to this method to implement
* backup correctly.
*/
char BeginToken() throws IOException; // SUPPRESS CHECKSTYLE we'll rename it later
/**
* Returns a string made up of characters from the token mark up to
* to the current buffer position.
*/
String GetImage(); // SUPPRESS CHECKSTYLE we'll rename it later
/**
* Returns an array of characters that make up the suffix of length 'len' for
* the current token. This is used to build up the matched string
* for use in actions in the case of MORE. A simple and inefficient
* implementation of this is as follows :
*
* <pre>{@code
* String t = tokenImage();
* return t.substring(t.length() - len, t.length()).toCharArray();
* }</pre>
*
* @param len Length of the returned array
*
* @return The suffix
*
* @throws IndexOutOfBoundsException If len is greater than the length of the
* current token
*/
char[] GetSuffix(int len); // SUPPRESS CHECKSTYLE we'll rename it later
/**
* Pushes a given number of already read chars into the buffer.
* Subsequent calls to {@link #readChar()} will read those characters
* before proceeding to read the underlying char stream.
*
* <p>A lexer calls this method if it has already read some characters,
* but cannot use them to match a (longer) token. So, they will
* be used again as the prefix of the next token.
*
* @throws AssertionError If the requested amount is greater than the
* number of read chars
*/
void backup(int amount);
@Deprecated
int getBeginColumn();
@Deprecated
int getBeginLine();
/** Returns the column number of the last character for the current token. */
int getEndColumn();
/** Returns the line number of the last character for current token. */
int getEndLine();
// These methods are added by PMD
/**
* Returns the token document for the tokens being built. Having it
* here is the most convenient place for the time being.
*/
default JavaccTokenDocument getTokenDocument() {
return null; // for VelocityCharStream
}
/** Returns the start offset of the current token (in the original source), inclusive. */
default int getStartOffset() {
return -1;
}
/** Returns the end offset of the current token (in the original source), exclusive. */
default int getEndOffset() {
return -1;
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java
+18 -1
View File
@@ -6,6 +6,7 @@ package net.sourceforge.pmd.lang.ast;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.checkerframework.checker.nullness.qual.NonNull;
import net.sourceforge.pmd.lang.document.TextFile;
@@ -39,7 +40,7 @@ public class FileAnalysisException extends RuntimeException {
super(message, cause);
}
FileAnalysisException setFileName(String filename) {
public FileAnalysisException setFileName(String filename) {
this.filename = Objects.requireNonNull(filename);
return this;
}
@@ -55,6 +56,22 @@ public class FileAnalysisException extends RuntimeException {
return filename;
}
@Override
public String getMessage() {
return errorKind() + StringUtils.uncapitalize(positionToString()) + ": " + super.getMessage();
}
protected String errorKind() {
return "Error";
}
protected String positionToString() {
if (hasFileName()) {
return " in file '" + getFileName() + "'";
}
return "";
}
/**
* Wraps the cause into an analysis exception. If it is itself an analysis
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java
+25 -2
View File
@@ -6,8 +6,11 @@ package net.sourceforge.pmd.lang.ast;
import java.util.Iterator;
import org.apache.commons.lang3.StringUtils;
import net.sourceforge.pmd.annotation.Experimental;
import net.sourceforge.pmd.internal.util.IteratorUtil;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextRegion;
import net.sourceforge.pmd.reporting.Reportable;
@@ -37,18 +40,37 @@ public interface GenericToken<T extends GenericToken<T>> extends Comparable<T>,
T getPreviousComment();
/**
* Returns the token's text.
* Returns the token's text as a string.
*/
default String getImage() {
return getImageCs().toString();
}
/**
* Returns the image as a {@link CharSequence}.
* Returns the text of the token as a char sequence.
* This should be preferred when you can use eg {@link StringUtils}
* to do some processing, without having to create a string.
*/
CharSequence getImageCs();
/**
* Returns true if the image of this token equals
* the given charsequence. This does not create a
* string.
*
* @param charSeq A character sequence
*/
default boolean imageEquals(CharSequence charSeq) {
CharSequence imageCs = getImageCs();
if (imageCs instanceof Chars) {
return ((Chars) imageCs).contentEquals(charSeq);
}
return StringUtils.equals(imageCs, charSeq);
}
/** Returns a text region with the coordinates of this token. */
TextRegion getRegion();
@@ -58,6 +80,7 @@ public interface GenericToken<T extends GenericToken<T>> extends Comparable<T>,
*/
boolean isEof();
/**
* Returns true if this token is implicit, ie was inserted artificially
* and has a zero-length image.
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java
+1 -1
View File
@@ -132,7 +132,7 @@ public interface Node extends Reportable {
// Those are kept here because they're handled specially as XPath
// attributes
// attributes, for now
@Override
default int getBeginLine() {
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/ParseException.java
+5 -5
View File
@@ -40,11 +40,6 @@ public class ParseException extends FileAnalysisException {
this.currentToken = null;
}
public ParseException(String message, Throwable cause) {
super(message, cause);
this.currentToken = null;
}
public ParseException(String message, JavaccToken token) {
super(message);
this.currentToken = token;
@@ -59,6 +54,11 @@ public class ParseException extends FileAnalysisException {
currentToken = currentTokenVal;
}
@Override
protected String errorKind() {
return "Parse exception";
}
/**
* It uses "currentToken" and "expectedTokenSequences" to generate a parse
* error message and returns it. If this object has been created
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Parser.java
+38 -7
View File
@@ -46,21 +46,28 @@ public interface Parser {
private final SemanticErrorReporter reporter;
private final ClassLoader auxclasspathClassLoader;
private final PropertySource propertySource;
private final ParserTaskProperties propertySource;
public ParserTask(TextDocument textDoc, SemanticErrorReporter reporter, ClassLoader auxclasspathClassLoader) {
this.textDoc = Objects.requireNonNull(textDoc, "Text document was null");
this.reporter = Objects.requireNonNull(reporter, "reporter was null");
this.auxclasspathClassLoader = Objects.requireNonNull(auxclasspathClassLoader, "auxclasspathClassLoader was null");
this.propertySource = new ParserTaskProperties();
propertySource.definePropertyDescriptor(COMMENT_MARKER);
this(textDoc, reporter, new ParserTaskProperties(), auxclasspathClassLoader);
}
public ParserTask(TextDocument textDoc, SemanticErrorReporter reporter) {
this(textDoc, reporter, Parser.class.getClassLoader());
}
private ParserTask(TextDocument textDoc,
SemanticErrorReporter reporter,
ParserTaskProperties source,
ClassLoader auxclasspathClassLoader) {
this.textDoc = Objects.requireNonNull(textDoc, "Text document was null");
this.reporter = Objects.requireNonNull(reporter, "reporter was null");
this.auxclasspathClassLoader = Objects.requireNonNull(auxclasspathClassLoader, "auxclasspathClassLoader was null");
this.propertySource = new ParserTaskProperties(source);
}
public static final PropertyDescriptor<String> COMMENT_MARKER =
PropertyFactory.stringProperty("suppressionCommentMarker")
.desc("deprecated! NOPMD")
@@ -117,9 +124,33 @@ public interface Parser {
return getProperties().getProperty(COMMENT_MARKER);
}
/**
* Replace the text document with another.
*/
public ParserTask withTextDocument(TextDocument doc) {
return new ParserTask(doc, this.reporter, this.propertySource, this.auxclasspathClassLoader);
}
private static final class ParserTaskProperties extends AbstractPropertySource {
ParserTaskProperties() {
definePropertyDescriptor(COMMENT_MARKER);
}
ParserTaskProperties(ParserTaskProperties toCopy) {
for (PropertyDescriptor<?> prop : toCopy.getPropertyDescriptors()) {
definePropertyDescriptor(prop);
}
toCopy.getOverriddenPropertyDescriptors().forEach(
prop -> copyProperty(prop, toCopy, this)
);
}
static <T> void copyProperty(PropertyDescriptor<T> prop, PropertySource source, PropertySource target) {
target.setProperty(prop, source.getProperty(prop));
}
@Override
protected String getPropertySourceType() {
return "ParserOptions";
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java
+28 -10
View File
@@ -4,6 +4,8 @@
package net.sourceforge.pmd.lang.ast;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.document.TextRegion;
import net.sourceforge.pmd.lang.rule.xpath.NoAttribute;
@@ -17,23 +19,39 @@ public interface TextAvailableNode extends Node {
/**
* Returns the exact region of text delimiting
* the node in the underlying text document. Note
* that {@link #getReportLocation()} does not need
* to match this region. {@link #getReportLocation()}
* can be scoped down to a specific token, eg the
* class identifier.
* Returns the exact region of text delimiting the node in the underlying
* text document. Note that {@link #getReportLocation()} does not need
* to match this region. {@link #getReportLocation()} can be scoped down
* to a specific token, eg the class identifier. This region uses
* the translated coordinate system, ie the coordinate system of
* {@link #getTextDocument()}.
*/
@Override
TextRegion getTextRegion();
/**
* Returns the original source code underlying this node. In
* particular, for a {@link RootNode}, returns the whole text
* of the file.
* Returns the original source code underlying this node, before
* any escapes have been translated. In particular, for a {@link RootNode},
* returns the whole text of the file.
*
* @see TextDocument#sliceOriginalText(TextRegion)
*/
@NoAttribute
CharSequence getText();
default Chars getOriginalText() {
return getTextDocument().sliceOriginalText(getTextRegion());
}
/**
* Returns the source code underlying this node, after any escapes
* have been translated. In particular, for a {@link RootNode}, returns
* the whole text of the file.
*
* @see TextDocument#sliceTranslatedText(TextRegion)
*/
@NoAttribute
default Chars getText() {
return getTextDocument().sliceTranslatedText(getTextRegion());
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java
+6 -4
View File
@@ -53,12 +53,14 @@ public final class TokenMgrError extends FileAnalysisException {
return column;
}
@Override
protected String positionToString() {
return super.positionToString() + " at line " + line + ", column " + column;
}
@Override
public String getMessage() {
String leader = hasFileName() ? "Lexical error in file " + getFileName() : "Lexical error";
return leader + " at line " + line + ", column " + column + ". Encountered: " + super.getMessage();
protected String errorKind() {
return "Lexical error";
}
/**
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java
-6
View File
@@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.annotation.Experimental;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.impl.AbstractNode;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.FileLocation;
import net.sourceforge.pmd.lang.document.TextRegion;
import net.sourceforge.pmd.util.StringUtil;
@@ -48,11 +47,6 @@ public abstract class AbstractJjtreeNode<B extends AbstractJjtreeNode<B, N>, N e
this.image = image;
}
@Override
public final Chars getText() {
return getTextDocument().sliceText(getTextRegion());
}
@Override
public final TextRegion getTextRegion() {
return TextRegion.fromBothOffsets(getFirstToken().getStartOffset(),
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/AbstractTokenManager.java → pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractTokenManager.java
+1 -4
View File
@@ -2,19 +2,16 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast;
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.util.HashMap;
import java.util.Map;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
/**
* A base class for the token managers generated by JavaCC.
*
* TODO move to impl.javacc package
*/
public abstract class AbstractTokenManager implements TokenManager<JavaccToken> {
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeTranslator.java
+71
View File
@@ -0,0 +1,71 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import static java.lang.Integer.min;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* A base class for readers that handle escapes starting with a backslash.
*/
public abstract class BackslashEscapeTranslator extends EscapeTranslator {
private static final char BACKSLASH = '\\';
/**
* An offset until which we read backslashes and decided they were not
* an escape. The read procedure may cut off in the middle of the escape,
* and turn an even num of backslashes into an odd one, so until we crossed
* this offset, backslashes are not treated specially.
*/
private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE;
public BackslashEscapeTranslator(TextDocument builder) {
super(builder);
}
@Override
protected int gobbleMaxWithoutEscape(final int maxOff) throws MalformedSourceException {
int off = this.bufpos;
boolean seenBackslash = true;
int notEscapeEnd = this.savedNotEscapeSpecialEnd;
while (off < maxOff) {
seenBackslash = input.charAt(off) == BACKSLASH && notEscapeEnd >= off;
if (seenBackslash) {
break;
}
off++;
}
if (!seenBackslash || off == maxOff) {
this.bufpos = off;
return off;
}
return handleBackslash(maxOff, off);
}
protected abstract int handleBackslash(int maxOff, int firstBackslashOff) throws MalformedSourceException;
@Override
protected int recordEscape(int startOffsetInclusive, int endOffsetExclusive, Chars translation) {
this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE;
return super.recordEscape(startOffsetInclusive, endOffsetExclusive, translation);
}
protected int abortEscape(int off, int maxOff) {
// not an escape sequence
int min = min(maxOff, off);
// save the number of backslashes that are part of the escape,
// might have been cut in half by the maxReadahead
this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE;
this.bufpos = min;
return min;
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java
+174
View File
@@ -0,0 +1,174 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.io.EOFException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.FileLocation;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.document.TextRegion;
/**
* PMD flavour of character streams used by JavaCC parsers.
*/
public final class CharStream {
private final JavaccTokenDocument tokenDoc;
private final TextDocument textDoc;
private final Chars chars;
private final boolean useMarkSuffix;
private int curOffset;
private int markOffset;
private CharStream(JavaccTokenDocument tokenDoc) {
this.tokenDoc = tokenDoc;
this.textDoc = tokenDoc.getTextDocument();
this.chars = textDoc.getText();
this.useMarkSuffix = tokenDoc.useMarkSuffix();
}
/**
* Create a new char stream for the given document. This may create
* a new {@link TextDocument} view over the original, which reflects
* its character escapes.
*/
public static CharStream create(TextDocument doc, TokenDocumentBehavior behavior) throws MalformedSourceException {
TextDocument translated = behavior.translate(doc);
return new CharStream(new JavaccTokenDocument(translated, behavior));
}
/**
* Returns the next character from the input. After a {@link #backup(int)},
* some of the already read chars must be spit out again.
*
* @return The next character
*
* @throws EOFException Upon EOF
*/
public char readChar() throws EOFException {
if (curOffset == chars.length()) {
throw new EOFException();
}
return chars.charAt(curOffset++);
}
/**
* Calls {@link #readChar()} and returns its value, marking its position
* as the beginning of the next token. All characters must remain in
* the buffer between two successive calls to this method to implement
* backup correctly.
*/
public char markTokenStart() throws EOFException {
markOffset = curOffset;
return readChar();
}
/**
* Returns a string made up of characters from the token mark up to
* to the current buffer position.
*/
public String getTokenImage() {
return getTokenImageCs().toString();
}
/**
* Returns a string made up of characters from the token mark up to
* to the current buffer position.
*/
public Chars getTokenImageCs() {
assert markOffset >= 0;
return chars.slice(markOffset, markLen());
}
private int markLen() {
return curOffset - markOffset;
}
/**
* Appends the suffix of length 'len' of the current token to the given
* string builder. This is used to build up the matched string
* for use in actions in the case of MORE.
*
* @param len Length of the returned array
*
* @throws IndexOutOfBoundsException If len is greater than the length of the current token
*/
public void appendSuffix(StringBuilder sb, int len) {
if (useMarkSuffix) {
assert len <= markLen() : "Suffix is greater than the mark length? " + len + " > " + markLen();
chars.appendChars(sb, curOffset - len, len);
} // otherwise dead code, kept because Javacc's argument expressions do side effects
}
/**
* Pushes a given number of already read chars into the buffer.
* Subsequent calls to {@link #readChar()} will read those characters
* before proceeding to read the underlying char stream.
*
* <p>A lexer calls this method if it has already read some characters,
* but cannot use them to match a (longer) token. So, they will
* be used again as the prefix of the next token.
*
* @throws AssertionError If the requested amount is greater than the
* length of the mark
*/
public void backup(int amount) {
if (amount > markLen()) {
throw new IllegalArgumentException();
}
curOffset -= amount;
}
/**
* Returns the column number of the last character for the current token.
* This is only used for parse exceptions and is very inefficient.
*/
public int getEndColumn() {
return endLocation().getEndColumn();
}
/**
* Returns the line number of the last character for current token.
* This is only used for parse exceptions and is very inefficient.
*/
public int getEndLine() {
return endLocation().getEndLine();
}
private FileLocation endLocation() {
return textDoc.toLocation(TextRegion.caretAt(getEndOffset()));
}
/** Returns the start offset of the current token (in the translated source), inclusive. */
public int getStartOffset() {
return markOffset;
}
/** Returns the end offset of the current token (in the translated source), exclusive. */
public int getEndOffset() {
return curOffset;
}
/**
* Returns the token document for the tokens being built. Having it
* here is the most convenient place for the time being.
*/
public JavaccTokenDocument getTokenDocument() {
return tokenDoc;
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java
-49
View File
@@ -1,49 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.util.function.Function;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.document.TextDocument;
public final class CharStreamFactory {
private CharStreamFactory() {
// util class
}
/**
* A char stream that doesn't perform any escape translation.
*/
public static CharStream simpleCharStream(TextDocument input) {
return simpleCharStream(input, JavaccTokenDocument::new);
}
/**
* A char stream that doesn't perform any escape translation.
*/
public static CharStream simpleCharStream(TextDocument input,
Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker) {
JavaccTokenDocument document = documentMaker.apply(input);
return new SimpleCharStream(document);
}
/**
* A char stream that translates java unicode sequences.
*/
public static CharStream javaCharStream(TextDocument input) {
return javaCharStream(input, JavaccTokenDocument::new);
}
/**
* A char stream that translates java unicode sequences.
*/
public static CharStream javaCharStream(TextDocument input, Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker) {
JavaccTokenDocument document = documentMaker.apply(input);
return new JavaCharStream(document);
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTranslator.java
+155
View File
@@ -0,0 +1,155 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import static java.lang.Integer.min;
import net.sourceforge.pmd.internal.util.AssertionUtil;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.FileLocation;
import net.sourceforge.pmd.lang.document.FragmentedDocBuilder;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* An object that can translate an input document into an output document,
* typically by replacing escape sequences with the character they represent.
*
* <p>This is an abstract class because the default implementation does not
* perform any escape processing. Subclasses refine this behavior.
*/
@SuppressWarnings("PMD.AssignmentInOperand")
public abstract class EscapeTranslator {
// Note that this can easily be turned into a java.io.Reader with
// efficient block IO, optimized for the common case where there are
// few or no escapes. This is part of the history of this file, but
// was removed for simplicity.
/**
* Source characters. When there is an escape, eg \ u00a0, the
* first backslash is replaced with the translated value of the
* escape. The bufpos is updated so that we read the next char
* after the escape.
*/
protected Chars input;
/** Position of the next char to read in the input. */
protected int bufpos;
/** Keep track of adjustments to make to the offsets, caused by unicode escapes. */
final FragmentedDocBuilder builder;
private Chars curEscape;
private int offInEscape;
/**
* Create a translator that will read from the given document.
*
* @param original Original document
*
* @throws NullPointerException If the parameter is null
*/
public EscapeTranslator(TextDocument original) {
AssertionUtil.requireParamNotNull("builder", original);
this.input = original.getText();
this.bufpos = 0;
this.builder = new FragmentedDocBuilder(original);
}
/**
* Translate all the input in the buffer. This consumes this object.
*
* @return The translated text document. If there is no escape, returns the original text
*
* @throws IllegalStateException If this method is called more than once on the same object
* @throws MalformedSourceException If there are invalid escapes in the source
*/
public TextDocument translateDocument() throws MalformedSourceException {
ensureOpen();
try {
return translateImpl();
} finally {
close();
}
}
private TextDocument translateImpl() {
if (this.bufpos == input.length()) {
return builder.build();
}
final int len = input.length(); // remove Integer.MAX_VALUE
int readChars = 0;
while (readChars < len && (this.bufpos < input.length() || curEscape != null)) {
if (curEscape != null) {
int toRead = min(len - readChars, curEscape.length() - offInEscape);
readChars += toRead;
offInEscape += toRead;
if (curEscape.length() == offInEscape) {
curEscape = null;
continue;
} else {
break; // len cut us off, we'll retry next time
}
}
int bpos = this.bufpos;
int nextJump = gobbleMaxWithoutEscape(min(input.length(), bpos + len - readChars));
int newlyReadChars = nextJump - bpos;
assert newlyReadChars >= 0 && (readChars + newlyReadChars) <= len;
if (newlyReadChars == 0 && nextJump == input.length()) {
// eof
break;
}
readChars += newlyReadChars;
}
return builder.build();
}
/**
* Returns the max offset, EXclusive, up to which we can cut the input
* array from the bufpos to dump it into the output array.
*
* @param maxOff Max offset up to which to read ahead
*/
protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException {
this.bufpos = maxOff;
return maxOff;
}
protected int recordEscape(final int startOffsetInclusive, int endOffsetExclusive, Chars translation) {
assert endOffsetExclusive > startOffsetInclusive && startOffsetInclusive >= 0;
this.builder.recordDelta(startOffsetInclusive, endOffsetExclusive, translation);
this.bufpos = endOffsetExclusive;
this.curEscape = translation;
this.offInEscape = 0;
return startOffsetInclusive;
}
/**
* Closing a translator does not close the underlying document, it just
* clears the intermediary state.
*/
private void close() {
this.bufpos = -1;
this.input = null;
}
/** Check to make sure that the stream has not been closed */
protected final void ensureOpen() {
if (input == null) {
throw new IllegalStateException("Closed");
}
}
protected FileLocation locationAt(int indexInInput) {
return builder.toLocation(indexInInput);
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaCharStream.java
-119
View File
@@ -1,119 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.io.EOFException;
import java.io.IOException;
import net.sourceforge.pmd.lang.document.Chars;
/**
* This stream buffers the whole file in memory before parsing,
* and track start/end offsets of tokens. This allows building {@link JavaccToken}.
* The buffer is assumed to be composed of only ASCII characters,
* and the stream unescapes Unicode escapes. The {@link #getTokenDocument() token document}
* stores the original file with escapes and all.
*/
public class JavaCharStream extends JavaCharStreamBase {
// full text with nothing escaped and all
private final Chars fullText;
private final JavaccTokenDocument document;
private int[] startOffsets;
public JavaCharStream(JavaccTokenDocument document) {
super(document.getTextDocument().newReader());
this.fullText = document.getFullText();
this.document = document;
this.startOffsets = new int[bufsize];
maxNextCharInd = fullText.length();
nextCharBuf = null;
}
@Override
protected void ExpandBuff(boolean wrapAround) {
int[] newStartOffsets = new int[bufsize + 2048];
if (wrapAround) {
System.arraycopy(startOffsets, tokenBegin, newStartOffsets, 0, bufsize - tokenBegin);
System.arraycopy(startOffsets, 0, newStartOffsets, bufsize - tokenBegin, bufpos);
startOffsets = newStartOffsets;
} else {
System.arraycopy(startOffsets, tokenBegin, newStartOffsets, 0, bufsize - tokenBegin);
startOffsets = newStartOffsets;
}
super.ExpandBuff(wrapAround);
}
@Override
protected void UpdateLineColumn(char c) {
startOffsets[bufpos] = nextCharInd;
super.UpdateLineColumn(c);
}
@Override
public int getStartOffset() {
return startOffsets[tokenBegin];
}
@Override
public int getEndOffset() {
if (isAtEof()) {
return fullText.length();
} else {
return startOffsets[bufpos] + 1; // + 1 for exclusive
}
}
@Override
public JavaccTokenDocument getTokenDocument() {
return document;
}
@Override
public String GetImage() {
if (bufpos >= tokenBegin) {
return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
} else {
return new String(buffer, tokenBegin, bufsize - tokenBegin)
+ new String(buffer, 0, bufpos + 1);
}
}
@Override
protected char ReadByte() throws IOException {
++nextCharInd;
if (isAtEof()) {
if (bufpos != 0) {
--bufpos;
if (bufpos < 0) {
bufpos += bufsize;
}
} else {
bufline[bufpos] = line;
bufcolumn[bufpos] = column;
startOffsets[bufpos] = fullText.length();
}
throw new EOFException();
}
return fullText.charAt(nextCharInd);
}
private boolean isAtEof() {
return nextCharInd >= fullText.length();
}
@Override
protected void FillBuff() {
throw new IllegalStateException("Buffer shouldn't be refilled");
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java
+94
View File
@@ -0,0 +1,94 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* An implementation of {@link EscapeTranslator} that translates Java
* unicode escapes.
*/
@SuppressWarnings("PMD.AssignmentInOperand")
public final class JavaEscapeTranslator extends BackslashEscapeTranslator {
public JavaEscapeTranslator(TextDocument input) {
super(input);
}
@Override
protected int handleBackslash(final int maxOff, final int firstBackslashOff) throws MalformedSourceException {
int off = firstBackslashOff;
while (off < input.length() && input.charAt(off) == '\\') {
off++;
}
int bslashCount = off - firstBackslashOff;
// is there an escape at offset firstBslashOff?
if ((bslashCount & 1) == 1 // odd number of backslashes
&& off < input.length() && input.charAt(off) == 'u') { // at least one 'u'
// this is enough to expect an escape or throw an exception
while (off < input.length() && input.charAt(off) == 'u') {
// consume all the 'u's
off++;
}
Chars value = escapeValue(firstBackslashOff, off - 1);
int endOffset = off + 4; // + 4 hex digits
return recordEscape(firstBackslashOff, endOffset, value);
} else {
return abortEscape(off, maxOff);
}
}
private Chars escapeValue(int posOfFirstBackSlash, final int offOfTheU) throws MalformedSourceException {
int off = offOfTheU;
try {
char c = (char)
( hexVal(input.charAt(++off)) << 12 // SUPPRESS CHECKSTYLE paren pad
| hexVal(input.charAt(++off)) << 8
| hexVal(input.charAt(++off)) << 4
| hexVal(input.charAt(++off))
);
return Chars.wrap(Character.toString(c));
} catch (NumberFormatException | IndexOutOfBoundsException e) {
// cut off u and 4 digits
String escape = input.substring(offOfTheU, Math.min(input.length(), offOfTheU + 5));
throw new MalformedSourceException("Invalid unicode escape \\" + escape, e, locationAt(posOfFirstBackSlash));
}
}
private static int hexVal(char c) {
switch (c) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return c - '0';
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
return c - ('A' - 10);
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
return c - ('a' - 10);
default:
throw new NumberFormatException("Character '" + c + "' is not a valid hexadecimal digit");
}
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java
+32 -28
View File
@@ -4,8 +4,8 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.FileLocation;
import net.sourceforge.pmd.lang.document.TextRegion;
@@ -76,6 +76,19 @@ public class JavaccToken implements GenericToken<JavaccToken> {
public JavaccToken specialToken;
// common constructor, with a CharSequence parameter
JavaccToken(int kind, CharSequence image, int startInclusive, int endExclusive, JavaccTokenDocument document) {
assert document != null : "Null document";
assert image instanceof String || image instanceof Chars : "Null image";
assert TextRegion.isValidRegion(startInclusive, endExclusive, document.getTextDocument());
this.kind = kind;
this.image = image;
this.startOffset = startInclusive;
this.endOffset = endExclusive;
this.document = document;
}
/**
* Builds a new token of the specified kind.
*
@@ -85,19 +98,15 @@ public class JavaccToken implements GenericToken<JavaccToken> {
* @param endExclusive End of the token in the text file (before translating escapes)
* @param document Document owning the token
*/
public JavaccToken(int kind,
CharSequence image,
int startInclusive,
int endExclusive,
JavaccTokenDocument document) {
assert document != null : "Null document";
assert TextRegion.isValidRegion(startInclusive, endExclusive, document.getTextDocument());
public JavaccToken(int kind, Chars image, int startInclusive, int endExclusive, JavaccTokenDocument document) {
this(kind, (CharSequence) image, startInclusive, endExclusive, document);
}
this.kind = kind;
this.image = image;
this.startOffset = startInclusive;
this.endOffset = endExclusive;
this.document = document;
/**
* Constructor with a {@link String} image (see {@link #JavaccToken(int, Chars, int, int, JavaccTokenDocument) the other ctor}).
*/
public JavaccToken(int kind, String image, int startInclusive, int endExclusive, JavaccTokenDocument document) {
this(kind, (CharSequence) image, startInclusive, endExclusive, document);
}
/**
@@ -128,12 +137,18 @@ public class JavaccToken implements GenericToken<JavaccToken> {
}
@Override
public CharSequence getImageCs() {
return image;
public Chars getImageCs() {
// wrap it: it's zero cost (images are either Chars or String) and Chars has a nice API
return Chars.wrap(image);
}
@Override
public TextRegion getRegion() {
public String getImage() {
return image.toString();
}
@Override
public final TextRegion getRegion() {
return TextRegion.fromBothOffsets(startOffset, endOffset);
}
@@ -171,24 +186,13 @@ public class JavaccToken implements GenericToken<JavaccToken> {
public JavaccToken replaceImage(CharStream charStream) {
return new JavaccToken(
this.kind,
charStream.GetImage(),
charStream.getTokenImageCs(),
this.startOffset,
charStream.getEndOffset(),
this.document
);
}
public JavaccToken withImage(String image) {
return new JavaccToken(
this.kind,
image,
this.startOffset,
this.endOffset,
this.document
);
}
/**
* Returns a new token with the given kind, and all other parameters
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java
+124 -55
View File
@@ -4,23 +4,137 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.util.Collections;
import java.util.List;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* Token document for Javacc implementations. This is a helper object
* for generated token managers.
* for generated token managers. Note: the extension point is a custom
* implementation of {@link TokenDocumentBehavior}, see {@link JjtreeParserAdapter#tokenBehavior()},
* {@link JavaCCTokenizer#tokenBehavior()}
*/
public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
public final class JavaccTokenDocument extends TokenDocument<JavaccToken> {
private final TokenDocumentBehavior behavior;
private JavaccToken first;
public JavaccTokenDocument(TextDocument textDocument) {
public JavaccTokenDocument(TextDocument textDocument, TokenDocumentBehavior behavior) {
super(textDocument);
this.behavior = behavior;
}
/**
* Overridable configuration of a token document.
*/
public static class TokenDocumentBehavior {
public static final TokenDocumentBehavior DEFAULT = new TokenDocumentBehavior(Collections.emptyList());
private final List<String> tokenNames;
public TokenDocumentBehavior(List<String> tokenNames) {
this.tokenNames = tokenNames;
}
/**
* Returns true if the lexer should accumulate the image of MORE
* tokens into the StringBuilder jjimage. This is useless in our
* current implementations, because the image of tokens can be cut
* out using text coordinates, so doesn't need to be put into a separate string.
* The default returns false, which makes {@link CharStream#appendSuffix(StringBuilder, int)} a noop.
*/
public boolean useMarkSuffix() {
return false;
}
/**
* Translate the escapes of the source document. The default implementation
* does not perform any escaping.
*
* @param text Source doc
*
* @see EscapeTranslator
*
* TODO move that to LanguageVersionHandler once #3919 (Merge CPD and PMD language) is implemented
*/
public TextDocument translate(TextDocument text) throws MalformedSourceException {
return text;
}
/**
* Returns a string that describes the token kind.
*
* @param kind Kind of token
*
* @return A descriptive string
*/
public final @NonNull String describeKind(int kind) {
if (kind == JavaccToken.IMPLICIT_TOKEN) {
return "<implicit token>";
}
String impl = describeKindImpl(kind);
if (impl != null) {
return impl;
}
return "<token of kind " + kind + ">";
}
/**
* Describe the given kind. If this returns a non-null value, then
* that's what {@link #describeKind(int)} will use. Otherwise a default
* implementation is used.
*
* <p>An implementation typically uses the JavaCC-generated array
* named {@code <parser name>Constants.tokenImage}. Remember to
* check the bounds of the array.
*
* @param kind Kind of token
*
* @return A descriptive string, or null to use default
*/
protected @Nullable String describeKindImpl(int kind) {
if (kind >= 0 && kind < tokenNames.size()) {
return tokenNames.get(kind);
}
return null;
}
/**
* Creates a new token with the given kind. This is called back to
* by JavaCC-generated token managers (jjFillToken). Note that a
* created token is not guaranteed to end up in the final token chain.
*
* @param kind Kind of the token
* @param cs Char stream of the file. This can be used to get text
* coordinates and the image
* @param image Shared instance of the image token. If this is non-null,
* then no call to {@link CharStream#getTokenImage()} should be
* issued.
*
* @return A new token
*/
public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream cs, @Nullable String image) {
return new JavaccToken(
kind,
image == null ? cs.getTokenImageCs() : image,
cs.getStartOffset(),
cs.getEndOffset(),
self
);
}
}
boolean useMarkSuffix() {
return behavior.useMarkSuffix();
}
/**
@@ -52,62 +166,17 @@ public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
}
/**
* Returns a string that describes the token kind.
*
* @param kind Kind of token
*
* @return A descriptive string
* @see TokenDocumentBehavior#describeKind(int)
*/
public final @NonNull String describeKind(int kind) {
if (kind == JavaccToken.IMPLICIT_TOKEN) {
return "<implicit token>";
}
String impl = describeKindImpl(kind);
if (impl != null) {
return impl;
}
return "<token of kind " + kind + ">";
public @NonNull String describeKind(int kind) {
return behavior.describeKind(kind);
}
/**
* Describe the given kind. If this returns a non-null value, then
* that's what {@link #describeKind(int)} will use. Otherwise a default
* implementation is used.
*
* <p>An implementation typically uses the JavaCC-generated array
* named {@code <parser name>Constants.tokenImage}. Remember to
* check the bounds of the array.
*
* @param kind Kind of token
*
* @return A descriptive string, or null to use default
*/
protected @Nullable String describeKindImpl(int kind) {
return null;
}
/**
* Creates a new token with the given kind. This is called back to
* by JavaCC-generated token managers (jjFillToken). Note that a
* created token is not guaranteed to end up in the final token chain.
*
* @param kind Kind of the token
* @param cs Char stream of the file. This can be used to get text
* coordinates and the image
* @param image Shared instance of the image token. If this is non-null,
* then no call to {@link CharStream#GetImage()} should be
* issued.
*
* @return A new token
* @see TokenDocumentBehavior#createToken(JavaccTokenDocument, int, CharStream, String)
*/
public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) {
return new JavaccToken(
kind,
image == null ? cs.GetImage() : image,
cs.getStartOffset(),
cs.getEndOffset(),
this
);
return behavior.createToken(this, kind, cs, image);
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java
-8
View File
@@ -6,8 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.lang.ast.TextAvailableNode;
import net.sourceforge.pmd.lang.ast.impl.GenericNode;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextRegion;
import net.sourceforge.pmd.reporting.Reportable;
/**
@@ -19,12 +17,6 @@ import net.sourceforge.pmd.reporting.Reportable;
*/
public interface JjtreeNode<N extends JjtreeNode<N>> extends GenericNode<N>, TextAvailableNode, Reportable {
@Override
Chars getText();
@Override
TextRegion getTextRegion();
// todo token accessors should most likely be protected in PMD 7.
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
+10 -13
View File
@@ -4,12 +4,10 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.Parser;
import net.sourceforge.pmd.lang.ast.RootNode;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* Base implementation of the {@link Parser} interface for JavaCC language
@@ -24,20 +22,19 @@ public abstract class JjtreeParserAdapter<R extends RootNode> implements Parser
// inheritance only
}
protected abstract JavaccTokenDocument newDocumentImpl(TextDocument textDocument);
protected CharStream newCharStream(JavaccTokenDocument tokenDocument) {
return new SimpleCharStream(tokenDocument);
}
protected abstract JavaccTokenDocument.TokenDocumentBehavior tokenBehavior();
@Override
public R parse(ParserTask task) throws ParseException {
JavaccTokenDocument doc = newDocumentImpl(task.getTextDocument());
CharStream charStream = newCharStream(doc);
public final R parse(ParserTask task) throws ParseException {
try {
// First read the source file and interpret escapes
CharStream charStream = CharStream.create(task.getTextDocument(), tokenBehavior());
// We replace the text document, so that it reflects escapes properly
// Escapes are processed by CharStream#create
task = task.withTextDocument(charStream.getTokenDocument().getTextDocument());
// Finally, do the parsing
return parseImpl(charStream, task);
} catch (TokenMgrError tme) {
} catch (FileAnalysisException tme) {
throw tme.setFileName(task.getFileDisplayName());
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/MalformedSourceException.java
+35
View File
@@ -0,0 +1,35 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.util.Objects;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.document.FileLocation;
/**
* A {@link FileAnalysisException} thrown when the source format is invalid,
* for example if some unicode escapes cannot be translated.
*/
public class MalformedSourceException extends FileAnalysisException {
private final FileLocation location;
public MalformedSourceException(String message, Throwable cause, FileLocation fileLocation) {
super(message, cause);
this.location = Objects.requireNonNull(fileLocation);
setFileName(fileLocation.getFileName());
}
@Override
protected String positionToString() {
return super.positionToString() + " at " + location.startPosToString();
}
@Override
protected String errorKind() {
return "Source format error";
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/SimpleCharStream.java
-20
View File
@@ -1,20 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc;
/**
* A char stream that does not perform unicode escaping.
*/
public class SimpleCharStream extends JavaCharStream {
public SimpleCharStream(JavaccTokenDocument document) {
super(document);
}
@Override
protected boolean doEscape() {
return false;
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java
+113
View File
@@ -0,0 +1,113 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.document;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.NonNull;
/**
* Base class for documents that apply a transform to their output offsets.
* This includes translated documents, and slices (subdocument views).
*/
abstract class BaseMappedDocument implements TextDocument {
protected final TextDocument base;
BaseMappedDocument(TextDocument base) {
this.base = base;
}
@Override
public long getCheckSum() {
return base.getCheckSum();
}
@Override
public String getPathId() {
return base.getPathId();
}
@Override
public String getDisplayName() {
return base.getDisplayName();
}
@Override
public Chars sliceOriginalText(TextRegion region) {
return base.sliceOriginalText(inputRegion(region));
}
@Override
public FileLocation toLocation(TextRegion region) {
return base.toLocation(inputRegion(region));
}
@Override
public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) {
// see the doc, lines do not need to be translated
return base.createLineRange(startLineInclusive, endLineInclusive);
}
@Override
public TextPos2d lineColumnAtOffset(int offset, boolean inclusive) {
return base.lineColumnAtOffset(inputOffset(offset, inclusive));
}
/**
* Translate a region given in the coordinate system of this
* document, to the coordinate system of the base document.
* This works as if creating a new region with both start and end
* offsets translated through {@link #inputOffset(int, boolean)}. The
* returned region may have a different length.
*
* @param outputRegion Output region
*
* @return Input region
*/
protected @NonNull TextRegion inputRegion(TextRegion outputRegion) {
return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true),
inputOffset(outputRegion.getEndOffset(), false));
}
/**
* Returns the input offset for the given output offset. This maps
* back an offset in the coordinate system of this document, to the
* coordinate system of the base document. This includes the
* length of any unicode escapes.
*
* <pre>
* input: "a\u00a0b" (original document)
* translated: "a b" (this document)
*
* translateOffset(0) = 0
* translateOffset(1) = 1
* translateOffset(2) = 7 // includes the length of the escape
* </pre>
*
* @param outOffset Output offset
* @param inclusive Whether the offset is to be interpreted as the index of a character (true),
* or the position after a character (false)
*
* @return Input offset
*/
protected final int inputOffset(int outOffset, boolean inclusive) {
if (outOffset < 0 || outOffset > getLength()) {
throw new IndexOutOfBoundsException();
}
return localOffsetTransform(outOffset, inclusive);
}
/**
* Output offset to input offset.
*/
protected abstract int localOffsetTransform(int outOffset, boolean inclusive);
@Override
public void close() throws IOException {
base.close();
}
}
pmd-core/src/main/java/net/sourceforge/pmd/lang/document/Chars.java
+10
View File
@@ -581,6 +581,16 @@ public final class Chars implements CharSequence {
return StreamSupport.stream(lines().spliterator(), false);
}
/**
* Returns a new stringbuilder containing the whole contents of this
* char sequence.
*/
public StringBuilder toStringBuilder() {
StringBuilder sb = new StringBuilder(length());
appendChars(sb);
return sb;
}
/**
* Returns a new reader for the whole contents of this char sequence.
pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileLocation.java
+16 -1
View File
@@ -138,7 +138,7 @@ public final class FileLocation {
}
/**
* Creates a new location from the given parameters.
* Creates a new location for a range of text.
*
* @throws IllegalArgumentException If the file name is null
* @throws IllegalArgumentException If any of the line/col parameters are strictly less than 1
@@ -155,6 +155,21 @@ public final class FileLocation {
end.getColumn());
}
/**
* Returns a new location that starts and ends at the same position.
*
* @param fileName File name
* @param line Line number
* @param column Column number
*
* @return A new location
*
* @throws IllegalArgumentException See {@link #range(String, int, int, int, int)}
*/
public static FileLocation caret(String fileName, int line, int column) {
return new FileLocation(fileName, line, column, line, column);
}
@Override
public String toString() {
Loaded 30 of 92 files, more files were not shown because too many files have changed in this diff. Show more