phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move charstream

Browse Source
This commit is contained in:
Clément Fournier
2020-09-02 05:16:32 +02:00
parent 51170633b0
commit 155a895a93
27 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
View File

@ -16,7 +16,7 @@ import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.CpdCompat;
import net.sourceforge.pmd.util.document.TextDocument;

13
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java → pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java
View File

@ -2,13 +2,13 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.ast.impl.javacc.io;
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.io.EOFException;
import java.io.IOException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.FileLocation;
import net.sourceforge.pmd.util.document.TextDocument;
@ -28,9 +28,9 @@ public final class CharStream {
private int curOffset;
private int markOffset = -1;
private CharStream(JavaccTokenDocument tokenDoc, TextDocument textDoc) {
private CharStream(JavaccTokenDocument tokenDoc) {
this.tokenDoc = tokenDoc;
this.textDoc = textDoc;
this.textDoc = tokenDoc.getTextDocument();
this.chars = textDoc.getText();
this.useMarkSuffix = tokenDoc.useMarkSuffix();
}
@ -39,9 +39,8 @@ public final class CharStream {
* Create a new char stream for the given document.
*/
public static CharStream create(JavaccTokenDocument doc) throws IOException, MalformedSourceException {
try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) {
return new CharStream(doc, reader.translate(doc.getTextDocument()));
}
doc.translate();
return new CharStream(doc);
}
/**

1
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java
View File

@ -5,7 +5,6 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.util.document.FileLocation;
import net.sourceforge.pmd.util.document.TextRegion;

10
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java
View File

@ -4,12 +4,14 @@
package net.sourceforge.pmd.lang.ast.impl.javacc;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
@ -22,6 +24,7 @@ public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
final StringPool stringPool = new StringPool();
private JavaccToken first;
private TextDocument translatedDocument;
public JavaccTokenDocument(TextDocument textDocument) {
super(textDocument);
@ -47,6 +50,11 @@ public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
return new EscapeAwareReader(text);
}
final void translate() throws IOException, MalformedSourceException {
try (EscapeAwareReader reader = newReader(getTextDocument().getText())) {
translatedDocument = reader.translate(getTextDocument());
}
}
/**
* Open the document. This is only meant to be used by a Javacc-generated

1
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
View File

@ -10,7 +10,6 @@ import net.sourceforge.pmd.lang.Parser;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.RootNode;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.util.document.TextDocument;
/**

3
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java
View File

@ -13,6 +13,7 @@ import java.io.Reader;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.internal.util.AssertionUtil;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.util.StringUtil;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
@ -60,7 +61,7 @@ public class EscapeAwareReader extends Reader {
/**
* Translate all the input in the buffer. This is fed to a cursor initialized to zero.
*/
TextDocument translate(TextDocument source) throws IOException, MalformedSourceException {
public TextDocument translate(TextDocument source) throws IOException, MalformedSourceException {
readUnchecked(null, 0, Integer.MAX_VALUE);
return escapes.build(source);
}

23
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java
View File

@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io;
import java.io.IOException;
import org.apache.commons.lang3.NotImplementedException;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.LanguageVersion;
@ -61,12 +60,11 @@ final class FragmentedDocBuilder {
this.curOffInInput = endInInput;
}
TextDocument build(TextDocument document) {
TextDocument build(TextDocument original) {
if (firstFragment == null) {
// No deltas in whole document, there's a single fragment
// This is the case for > 97% of Java files (source: OpenJDK)
Fragment fragment = new Fragment(null, mainBuf.length(), mainBuf);
return new FragmentedTextDocument(document, fragment, fragment);
return original;
} else {
if (curOffInInput < mainBuf.length()) {
// there's some text left between the last fragment and the end of the doc
@ -74,7 +72,7 @@ final class FragmentedDocBuilder {
Chars remainder = mainBuf.slice(curOffInInput, remLen);
lastFragment = new Fragment(lastFragment, remLen, remainder);
}
return new FragmentedTextDocument(document, firstFragment, lastFragment);
return new FragmentedTextDocument(original, firstFragment, lastFragment);
}
}
@ -123,6 +121,8 @@ final class FragmentedDocBuilder {
@Override
public int translateOffset(int outputOffset) {
// todo this would be pretty slow when there are many escapes
// we could check save the fragment last accessed and
return base.translateOffset(inputOffsetAt(outputOffset, firstFragment));
}
@ -153,7 +153,7 @@ final class FragmentedDocBuilder {
@Override
public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) {
throw new NotImplementedException("TODO");
return base.createLineRange(startLineInclusive, endLineInclusive);
}
@Override
@ -177,6 +177,8 @@ final class FragmentedDocBuilder {
@Nullable Fragment next;
private final int inLength;
private final int outStart;
private final int inStart;
Fragment(@Nullable Fragment prev, int inLength, Chars chars) {
this.chars = chars;
@ -184,6 +186,11 @@ final class FragmentedDocBuilder {
this.inLength = inLength;
if (prev != null) {
prev.next = this;
this.outStart = prev.outEnd();
this.inStart = prev.inEnd();
} else {
this.outStart = 0;
this.inStart = 0;
}
}
@ -192,7 +199,7 @@ final class FragmentedDocBuilder {
}
int outStart() {
return prev != null ? prev.outEnd() : 0;
return outStart;
}
int outLen() {
@ -204,7 +211,7 @@ final class FragmentedDocBuilder {
}
int inStart() {
return prev != null ? prev.inEnd() : 0;
return inStart;
}
int inLen() {

5
pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java
View File

@ -62,6 +62,11 @@ final class RootTextDocument extends BaseCloseable implements TextDocument {
backend.close();
}
@Override
public Chars getText() {
return content.getNormalizedText();
}
@Override
public FileLocation toLocation(TextRegion region) {
checkInRange(region);

1
pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java
View File

@ -16,6 +16,7 @@ import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersion;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;

2
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -13,7 +13,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;

2
pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java
View File

@ -11,7 +11,7 @@ import java.io.IOException;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.Test;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.util.document.CpdCompat;
import net.sourceforge.pmd.util.document.TextDocument;
import net.sourceforge.pmd.util.document.TextFile;

2
pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
View File

@ -15,7 +15,7 @@ import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.java.ast.InternalApiBridge;
import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds;
import net.sourceforge.pmd.util.document.TextDocument;

2
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.java.ast;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker;
import net.sourceforge.pmd.util.document.TextDocument;

2
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java
View File

@ -16,7 +16,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader;
import net.sourceforge.pmd.util.document.Chars;

2
pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
View File

@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5TokenKinds;

2
pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
View File

@ -8,7 +8,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader;
import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds;

2
pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java
View File

@ -9,7 +9,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.util.document.TextDocument;
/**

2
pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
View File

@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.matlab.ast.MatlabTokenKinds;

2
pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.modelica.ast.ModelicaTokenKinds;

2
pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java
View File

@ -4,7 +4,7 @@
package net.sourceforge.pmd.lang.modelica.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;

2
pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
View File

@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.objectivec.ast.ObjectiveCTokenKinds;

2
pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
View File

@ -9,7 +9,7 @@ import java.util.Properties;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.plsql.ast.PLSQLTokenKinds;
public class PLSQLTokenizer extends JavaCCTokenizer {

2
pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java
View File

@ -9,7 +9,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.util.document.TextDocument;
public class PLSQLParser extends JjtreeParserAdapter<ASTInput> {

2
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -10,7 +10,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds;

2
pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java
View File

@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

2
pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java
View File

@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vf.ast;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;

2
pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java
View File

@ -10,7 +10,7 @@ import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.util.document.TextDocument;
/**