phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor token documents

Browse Source 
Isolate static parts
This commit is contained in:
Clément Fournier
2020-09-02 08:41:23 +02:00
parent a01f1a3574
commit 3ee32effda
25 changed files with 296 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -7,19 +7,14 @@ package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.util.Properties;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.lang.cpp.ast.CppEscapeTranslator;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
import net.sourceforge.pmd.util.document.TextFileContent;
/**
* The C++ tokenizer.
@ -62,59 +57,22 @@ public class CPPTokenizer extends JavaCCTokenizer {
}
}
/**
* @param chars Normalized chars
*/
private CharSequence maybeSkipBlocks(Chars chars) {
if (!skipBlocks) {
return chars;
}
int i = 0;
int lastLineStart = 0;
boolean skip = false;
StringBuilder filtered = new StringBuilder(chars.length());
while (i < chars.length()) {
if (chars.charAt(i) == TextFileContent.NORMALIZED_LINE_TERM_CHAR) {
Chars lastLine = chars.subSequence(lastLineStart, i);
Chars trimmed = lastLine.trim();
if (trimmed.contentEquals(skipBlocksStart, true)) {
skip = true;
} else if (trimmed.contentEquals(skipBlocksEnd, true)) {
skip = false;
}
if (!skip) {
lastLine.appendChars(filtered);
}
// always add newline, to preserve line numbers
filtered.append(TextFileContent.NORMALIZED_LINE_TERM_CHAR);
lastLineStart = i + 1;
}
i++;
}
if (lastLineStart < i && !skip) {
chars.appendChars(filtered, lastLineStart, i - lastLineStart);
}
return filtered;
}
@Override
protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
textDoc = TextDocument.readOnlyString(maybeSkipBlocks(textDoc.getText()), textDoc.getDisplayName(), textDoc.getLanguageVersion());
return new JavaccTokenDocument(textDoc) {
protected TokenDocumentBehavior newTokenDoc() {
return new TokenDocumentBehavior(CppTokenKinds.TOKEN_NAMES) {
@Override
protected TextDocument translate(TextDocument text) throws MalformedSourceException {
if (skipBlocks) {
try (CppBlockSkipper translator = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd)) {
text = translator.translateDocument();
}
}
try (CppEscapeTranslator translator = new CppEscapeTranslator(text)) {
return translator.translateDocument();
}
}
@Override
protected @Nullable String describeKindImpl(int kind) {
return CppTokenKinds.describe(kind);
}
};
}

40
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java Normal file
View File

@ -0,0 +1,40 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
/**
*
*/
class CppBlockSkipper extends EscapeTranslator {
private final Pattern skipStart;
private final Pattern skipEnd;
public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) {
super(original);
skipStart = Pattern.compile("^" + Pattern.quote(skipStartMarker));
skipEnd = Pattern.compile("^" + Pattern.quote(skipEndMarker));
}
@Override
protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException {
Matcher start = skipStart.matcher(input).region(this.bufpos, maxOff);
if (start.find()) {
Matcher end = skipStart.matcher(input).region(start.end(), maxOff);
if (end.find()) {
return recordEscape(start.start(), end.end(), Chars.EMPTY);
}
}
return super.gobbleMaxWithoutEscape(maxOff);
}
}

2
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java → pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java
View File

@ -2,7 +2,7 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp.ast;
package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeTranslator;
import net.sourceforge.pmd.util.document.Chars;