hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Cleanup spec

Browse Source
This commit is contained in:
Clément Fournier
2020-09-02 06:07:39 +02:00
parent 155a895a93
commit a11f45e511
24 changed files with 110 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
javacc-wrapper.xml
View File

@ -407,7 +407,7 @@ public final class ${token-constants-name} \{${line.separator}
* be used as a basis for a CPD Tokenizer.
*/
@net.sourceforge.pmd.annotation.InternalApi
public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.io.CharStream cs) {
public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) {
return new %%%TOKEN_MGR_NAME%%%(cs);
}

2
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java
View File

@ -18,8 +18,8 @@ import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.util.document.TextDocument;
import net.sourceforge.pmd.util.document.CpdCompat;
import net.sourceforge.pmd.util.document.TextDocument;
/**
* Generic implementation of a {@link Tokenizer} useful to any Antlr grammar.

2
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
View File

@ -14,9 +14,9 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.CpdCompat;
import net.sourceforge.pmd.util.document.TextDocument;

2
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java
View File

@ -51,7 +51,7 @@ public abstract class AbstractJjtreeNode<B extends AbstractJjtreeNode<B, N>, N e
@Override
public Chars getText() {
return getTextDocument().sliceText(getTextRegion());
return getTextDocument().sliceOriginalText(getTextRegion());
}
@Override

15
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java
View File

@ -26,7 +26,7 @@ public final class CharStream {
private final Chars chars;
private final boolean useMarkSuffix;
private int curOffset;
private int markOffset = -1;
private int markOffset;
private CharStream(JavaccTokenDocument tokenDoc) {
this.tokenDoc = tokenDoc;
@ -120,9 +120,12 @@ public final class CharStream {
* be used again as the prefix of the next token.
*
* @throws AssertionError If the requested amount is greater than the
* number of read chars
* length of the mark
*/
public void backup(int amount) {
if (amount > markLen()) {
throw new IllegalArgumentException();
}
curOffset -= amount;
}
@ -149,15 +152,15 @@ public final class CharStream {
}
/** Returns the start offset of the current token (in the original source), inclusive. */
/** Returns the start offset of the current token (in the translated source), inclusive. */
public int getStartOffset() {
return textDoc.translateOffset(markOffset);
return markOffset;
}
/** Returns the end offset of the current token (in the original source), exclusive. */
/** Returns the end offset of the current token (in the translated source), exclusive. */
public int getEndOffset() {
return textDoc.translateOffset(curOffset);
return curOffset;
}

6
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java
View File

@ -50,6 +50,12 @@ public class JavaccTokenDocument extends TokenDocument<JavaccToken> {
return new EscapeAwareReader(text);
}
@Override
public TextDocument getTextDocument() {
return translatedDocument == null ? super.getTextDocument()
: translatedDocument;
}
final void translate() throws IOException, MalformedSourceException {
try (EscapeAwareReader reader = newReader(getTextDocument().getText())) {
translatedDocument = reader.translate(getTextDocument());

41
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.LanguageVersion;
@ -80,49 +81,44 @@ final class FragmentedDocBuilder {
return inputOffsetAt(outputOffset, firstFragment);
}
static int inputOffsetAt(int outputOffset, Fragment firstFragment) {
static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) {
Fragment f = firstFragment;
if (f == null) {
return outputOffset;
}
int sum = outputOffset;
while (f != null && f.inStart() < sum) {
sum += f.inLen();
while (f.next != null && f.inEnd() < outputOffset) {
f = f.next;
}
return sum;
return f.outToIn(outputOffset);
}
static class FragmentedTextDocument implements TextDocument {
static final class FragmentedTextDocument implements TextDocument {
private final Fragment firstFragment;
private final Chars text;
private final TextDocument base;
FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) {
assert firstFragment != lastFragment; // NOPMD
this.firstFragment = firstFragment;
this.text = toChars(firstFragment, lastFragment);
this.base = base;
}
private static Chars toChars(Fragment firstFragment, Fragment lastFragment) {
if (firstFragment == lastFragment) {
return firstFragment.getChars();
}
StringBuilder sb = new StringBuilder(lastFragment.outEnd());
Fragment f = firstFragment;
while (f.next != null) {
while (f != null) {
f.getChars().appendChars(sb);
f = f.next;
}
return Chars.wrap(sb);
}
@Override
public int translateOffset(int outputOffset) {
// todo this would be pretty slow when there are many escapes
// we could check save the fragment last accessed and
// todo this would be pretty slow when we're in the middle of some escapes
// we could check save the fragment last accessed to speed it up, and look forwards & backwards
return base.translateOffset(inputOffsetAt(outputOffset, firstFragment));
}
@ -156,10 +152,20 @@ final class FragmentedDocBuilder {
return base.createLineRange(startLineInclusive, endLineInclusive);
}
@Override
public Chars sliceOriginalText(TextRegion region) {
return base.sliceOriginalText(translateRegion(region));
}
@Override
public FileLocation toLocation(TextRegion region) {
return base.toLocation(TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()),
translateOffset(region.getEndOffset())));
return base.toLocation(translateRegion(region));
}
@Override
public @NonNull TextRegion translateRegion(TextRegion region) {
return TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()),
translateOffset(region.getEndOffset()));
}
@Override
@ -168,7 +174,10 @@ final class FragmentedDocBuilder {
}
}
/**
* A delta from the original text to the translated text. This maps
* a region of the original document to some new characters.
*/
static final class Fragment {
private final Chars chars;

10
pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java
View File

@ -116,6 +116,16 @@ final class RootTextDocument extends BaseCloseable implements TextDocument {
return outOffset;
}
@Override
public TextRegion translateRegion(TextRegion region) {
return region;
}
@Override
public Chars sliceOriginalText(TextRegion region) {
return getText().subSequence(region.getStartOffset(), region.getEndOffset());
}
private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)";
private static final String INVALID_LINE_RANGE = "Line range %d..%d is not in range of this document (%d lines) (line numbers are 1-based)";

38
pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java
View File

@ -57,11 +57,17 @@ public interface TextDocument extends Closeable {
Chars getText();
/**
* Returns a region of the {@linkplain #getText() text} as a character sequence.
* Returns a slice of the original text. Note that this is not the
* same as {@code getText().subsequence}, as if this document has
* translated escapes, the returned char slice will contain the
* untranslated escapes, whereas {@link #getText()} would return
* the translated characters.
*
* @param region A region, in the coordinate system of this document
*
* @return The slice of the original text that corresponds to the region
*/
default Chars sliceText(TextRegion region) {
return getText().subSequence(region.getStartOffset(), region.getEndOffset());
}
Chars sliceOriginalText(TextRegion region);
/**
@ -71,8 +77,31 @@ public interface TextDocument extends Closeable {
*/
long getChecksum();
/**
* Returns the input offset for the given output offset. This maps
* back an offset in the coordinate system of this document, to the
* coordinate system of the original document.
*
* @param outOffset Output offset
*
* @return Input offset
*/
int translateOffset(int outOffset);
/**
* Translate a region given in the the coordinate system of this
* document, to the coordinate system of the original document.
* This works as if creating a new region with both start and end
* offsets translated through {@link #translateOffset(int)}. The
* returned region may have a different length.
*
* @param region Output region
*
* @return Input region
*/
TextRegion translateRegion(TextRegion region);
/**
* Returns the length in characters of the {@linkplain #getText() text}.
@ -152,6 +181,7 @@ public interface TextDocument extends Closeable {
*
* @see TextFile#forCharSeq(CharSequence, String, LanguageVersion)
*/
@SuppressWarnings("PMD.CloseResource")
static TextDocument readOnlyString(@NonNull CharSequence source, @NonNull String filename, @NonNull LanguageVersion lv) {
TextFile textFile = TextFile.forCharSeq(source, filename, lv);
try {

1
pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextFileBuilder.java
View File

@ -16,6 +16,7 @@ import net.sourceforge.pmd.lang.LanguageVersion;
/**
* A builder for a new text file.
*/
@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass")
public abstract class TextFileBuilder {
protected final LanguageVersion languageVersion;

7
pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java
View File

@ -53,8 +53,8 @@ public class CharStreamImplTest {
try {
stream.readChar();
} catch (Exception e) {
assertEquals(stream.getStartOffset(), 0);
assertEquals(stream.getEndOffset(), 0);
assertEquals(0, stream.getStartOffset());
assertEquals(0, stream.getEndOffset());
throw e;
}
}
@ -162,8 +162,9 @@ public class CharStreamImplTest {
assertEquals('c', stream.markTokenStart());
assertEquals('d', stream.readChar());
stream.backup(2); // ok
expect.expect(IllegalArgumentException.class);
stream.backup(10);
stream.backup(1);
}
@Test

6
pmd-core/src/test/java/net/sourceforge/pmd/util/document/TextDocumentTest.java
View File

@ -43,7 +43,7 @@ public class TextDocumentTest {
TextDocument doc = TextDocument.readOnlyString("bonjour\ntristesse", dummyVersion);
TextRegion region = TextRegion.fromOffsetLength(0, "bonjour\n".length());
assertEquals("bonjour\n", doc.sliceText(region).toString());
assertEquals("bonjour\n", doc.sliceOriginalText(region).toString());
FileLocation withLines = doc.toLocation(region);
assertEquals(1, withLines.getBeginLine());
@ -60,7 +60,7 @@ public class TextDocumentTest {
// We consider it's part of the next line
TextRegion region = TextRegion.fromOffsetLength("bonjour\n".length(), 0);
assertEquals("", doc.sliceText(region).toString());
assertEquals("", doc.sliceOriginalText(region).toString());
FileLocation withLines = doc.toLocation(region);
@ -78,7 +78,7 @@ public class TextDocumentTest {
TextRegion region = TextRegion.fromOffsetLength("bonjour".length(), 1);
assertEquals("\n", doc.sliceText(region).toString());
assertEquals("\n", doc.sliceOriginalText(region).toString());
FileLocation withLines = doc.toLocation(region);

2
pmd-cpp/etc/grammar/Cpp.jj
View File

@ -32,7 +32,7 @@ options {
PARSER_BEGIN(CppParserImpl)
package net.sourceforge.pmd.lang.cpp.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public final class CppParserImpl {

7
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java
View File

@ -12,7 +12,6 @@ import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.lang.Parser.ParserTask;
import net.sourceforge.pmd.lang.ast.NodeStream;
import net.sourceforge.pmd.lang.ast.RootNode;
import net.sourceforge.pmd.lang.ast.impl.GenericNode;
@ -42,11 +41,11 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ja
return doc;
}
void addTaskInfo(ParserTask task) {
this.doc = task.getTextDocument();
void addTaskInfo(TextDocument translatedDoc) {
this.doc = translatedDoc;
}
void setComments(List<Comment> comments) {
void setComments(List<Comment> comments) {
this.comments = comments;
}

2
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java
View File

@ -40,7 +40,7 @@ public class JavaParser extends JjtreeParserAdapter<ASTCompilationUnit> {
ASTCompilationUnit acu = parser.CompilationUnit();
acu.setNoPmdComments(parser.getSuppressMap());
acu.addTaskInfo(task);
acu.addTaskInfo(cs.getTokenDocument().getTextDocument()); // this is the translated document != task.getTextDocument()
checker.check(acu);
return acu;
}

2
pmd-javascript/etc/grammar/Ecmascript5.jj
View File

@ -15,7 +15,7 @@ options {
PARSER_BEGIN(Ecmascript5ParserImpl)
package net.sourceforge.pmd.lang.ecmascript5.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class Ecmascript5ParserImpl {

2
pmd-jsp/etc/grammar/Jsp.jjt
View File

@ -30,7 +30,7 @@ options {
PARSER_BEGIN(JspParserImpl)
package net.sourceforge.pmd.lang.jsp.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;

2
pmd-matlab/etc/grammar/Matlab.jj
View File

@ -21,7 +21,7 @@ options {
PARSER_BEGIN(MatlabParserImpl)
package net.sourceforge.pmd.lang.matlab.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class MatlabParserImpl {

2
pmd-modelica/etc/grammar/Modelica.jjt
View File

@ -49,7 +49,7 @@ options {
PARSER_BEGIN(ModelicaParserImpl)
package net.sourceforge.pmd.lang.modelica.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
class ModelicaParserImpl {

2
pmd-objectivec/etc/grammar/ObjectiveC.jj
View File

@ -21,7 +21,7 @@ package net.sourceforge.pmd.lang.objectivec.ast;
import java.io.*;
import java.util.*;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**

2
pmd-plsql/etc/grammar/PLSQL.jjt
View File

@ -154,7 +154,7 @@ package net.sourceforge.pmd.lang.plsql.ast;
import java.io.*;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PLSQLParserImpl {

2
pmd-python/etc/grammar/Python.jj
View File

@ -17,7 +17,7 @@ PARSER_BEGIN(PythonParserImpl)
package net.sourceforge.pmd.lang.python.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PythonParserImpl {

2
pmd-visualforce/etc/grammar/Vf.jjt
View File

@ -13,7 +13,7 @@ options {
PARSER_BEGIN(VfParserImpl)
package net.sourceforge.pmd.lang.vf.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class VfParserImpl {

2
pmd-vm/etc/grammar/Vm.jjt
View File

@ -46,7 +46,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;