";
+ }
+
+ /**
+ * Describe the given kind. If this returns a non-null value, then
+ * that's what {@link #describeKind(int)} will use. Otherwise a default
+ * implementation is used.
+ *
+ * An implementation typically uses the JavaCC-generated array
+ * named {@code Constants.tokenImage}. Remember to
+ * check the bounds of the array.
+ *
+ * @param kind Kind of token
+ *
+ * @return A descriptive string, or null to use default
+ */
+ protected @Nullable String describeKindImpl(int kind) {
+ if (kind >= 0 && kind < tokenNames.size()) {
+ return tokenNames.get(kind);
+ }
+ return null;
+ }
+
+
+ /**
+ * Creates a new token with the given kind. This is called back to
+ * by JavaCC-generated token managers (jjFillToken). Note that a
+ * created token is not guaranteed to end up in the final token chain.
+ *
+ * @param kind Kind of the token
+ * @param cs Char stream of the file. This can be used to get text
+ * coordinates and the image
+ * @param image Shared instance of the image token. If this is non-null,
+ * then no call to {@link CharStream#getTokenImage()} should be
+ * issued.
+ *
+ * @return A new token
+ */
+ public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream cs, @Nullable String image) {
+ return new JavaccToken(
+ kind,
+ image == null ? cs.getTokenImageCs() : image,
+ cs.getStartOffset(),
+ cs.getEndOffset(),
+ self
+ );
+ }
+ }
+
/**
* Returns true if the lexer should accumulate the image of MORE
* tokens into the StringBuilder jjimage. This is useless in our
@@ -37,10 +153,12 @@ public class JavaccTokenDocument extends TokenDocument {
}
/**
- * Create new (possibly) escaping reader for the given text. The default
- * implementation doesn't do any escaping.
+ * Translate the escapes of the source document. The default implementation
+ * does not perform any escaping.
*
* @param text Source doc
+ *
+ * @see EscapeTranslator
*/
protected TextDocument translate(TextDocument text) throws MalformedSourceException {
return text;
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
index 71a9da0a25..02cd2dafd2 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
@@ -8,7 +8,6 @@ import net.sourceforge.pmd.lang.Parser;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.RootNode;
-import net.sourceforge.pmd.util.document.TextDocument;
/**
* Base implementation of the {@link Parser} interface for JavaCC language
@@ -23,14 +22,12 @@ public abstract class JjtreeParserAdapter implements Parser
// inheritance only
}
- protected abstract JavaccTokenDocument newDocumentImpl(TextDocument textDocument);
+ protected abstract JavaccTokenDocument.TokenDocumentBehavior tokenBehavior();
@Override
public R parse(ParserTask task) throws ParseException {
- JavaccTokenDocument doc = newDocumentImpl(task.getTextDocument());
-
try {
- CharStream charStream = CharStream.create(doc);
+ CharStream charStream = CharStream.create(task.getTextDocument(), tokenBehavior());
return parseImpl(charStream, task);
} catch (FileAnalysisException tme) {
throw tme.setFileName(task.getFileDisplayName());
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java
index ee9b1a55cb..1d7f46e158 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java
@@ -6,6 +6,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io;
import static java.lang.Integer.min;
+import java.util.function.Function;
+
import net.sourceforge.pmd.internal.util.AssertionUtil;
import net.sourceforge.pmd.util.StringUtil;
import net.sourceforge.pmd.util.document.Chars;
@@ -159,4 +161,13 @@ public abstract class EscapeTranslator implements AutoCloseable {
return StringUtil.columnNumberAt(input, idxInInput);
}
+
+ public static Function translatorFor(Function translatorMaker) {
+ return original -> {
+ try (EscapeTranslator translator = translatorMaker.apply(original)) {
+ return translator.translateDocument();
+ }
+ };
+ }
+
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java
index 9b60c69724..32ddd0a4e3 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java
@@ -109,14 +109,6 @@ public final class Chars implements CharSequence {
str.getChars(start, start + count, cbuf, dstBegin);
}
- public void getChars(int srcBegin, CharBuffer buffer, int count) {
- if (count == 0) {
- return;
- }
- int start = idx(srcBegin);
- str.getChars(start, start + count, cbuf, dstBegin);
- }
-
/**
* Appends the character range identified by offset and length into
* the string builder. This is much more efficient than calling
diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java
index ca032d6f82..725383344e 100644
--- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java
+++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java
@@ -9,6 +9,7 @@ import static org.junit.Assert.fail;
import java.io.EOFException;
import java.io.IOException;
+import java.util.Collections;
import org.junit.Rule;
import org.junit.Test;
@@ -17,7 +18,7 @@ import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersion;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.util.document.TextDocument;
public class CharStreamImplTest {
@@ -139,19 +140,12 @@ public class CharStreamImplTest {
}
public CharStream simpleCharStream(String abcd) {
- return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion)));
+ return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion), TokenDocumentBehavior.DEFAULT);
}
public CharStream javaCharStream(String abcd) {
- return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion)) {
-
- @Override
- protected TextDocument translate(TextDocument text) throws MalformedSourceException {
- try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) {
- return translator.translateDocument();
- }
- }
- });
+ return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion),
+ new TokenDocumentBehavior(Collections.emptyList(), EscapeTranslator.translatorFor(JavaEscapeTranslator::new)));
}
@Test
diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java
index 2db00f2546..b6922c8a23 100644
--- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java
+++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java
@@ -10,14 +10,17 @@ import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.Assert;
import org.junit.Test;
+import net.sourceforge.pmd.lang.DummyLanguageModule;
+import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.util.document.Chars;
+import net.sourceforge.pmd.util.document.TextDocument;
public class JavaEscapeReaderTest {
@NonNull
public JavaEscapeTranslator readString(String input) {
- return new JavaEscapeTranslator(Chars.wrap(input));
+ return new JavaEscapeTranslator(TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion());
}
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
index c3f6830508..3905f23f3a 100644
--- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
@@ -7,19 +7,14 @@ package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.util.Properties;
-import org.checkerframework.checker.nullness.qual.Nullable;
-
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
-import net.sourceforge.pmd.lang.cpp.ast.CppEscapeTranslator;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;
-import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
-import net.sourceforge.pmd.util.document.TextFileContent;
/**
* The C++ tokenizer.
@@ -62,59 +57,22 @@ public class CPPTokenizer extends JavaCCTokenizer {
}
}
- /**
- * @param chars Normalized chars
- */
- private CharSequence maybeSkipBlocks(Chars chars) {
- if (!skipBlocks) {
- return chars;
- }
-
- int i = 0;
- int lastLineStart = 0;
- boolean skip = false;
- StringBuilder filtered = new StringBuilder(chars.length());
- while (i < chars.length()) {
- if (chars.charAt(i) == TextFileContent.NORMALIZED_LINE_TERM_CHAR) {
- Chars lastLine = chars.subSequence(lastLineStart, i);
- Chars trimmed = lastLine.trim();
- if (trimmed.contentEquals(skipBlocksStart, true)) {
- skip = true;
- } else if (trimmed.contentEquals(skipBlocksEnd, true)) {
- skip = false;
- }
- if (!skip) {
- lastLine.appendChars(filtered);
- }
- // always add newline, to preserve line numbers
- filtered.append(TextFileContent.NORMALIZED_LINE_TERM_CHAR);
- lastLineStart = i + 1;
- }
- i++;
- }
- if (lastLineStart < i && !skip) {
- chars.appendChars(filtered, lastLineStart, i - lastLineStart);
- }
- return filtered;
- }
-
@Override
- protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
- textDoc = TextDocument.readOnlyString(maybeSkipBlocks(textDoc.getText()), textDoc.getDisplayName(), textDoc.getLanguageVersion());
- return new JavaccTokenDocument(textDoc) {
+ protected TokenDocumentBehavior newTokenDoc() {
+ return new TokenDocumentBehavior(CppTokenKinds.TOKEN_NAMES) {
@Override
protected TextDocument translate(TextDocument text) throws MalformedSourceException {
+ if (skipBlocks) {
+ try (CppBlockSkipper translator = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd)) {
+ text = translator.translateDocument();
+ }
+ }
try (CppEscapeTranslator translator = new CppEscapeTranslator(text)) {
return translator.translateDocument();
}
}
-
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return CppTokenKinds.describe(kind);
- }
};
}
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java
new file mode 100644
index 0000000000..694d5cd978
--- /dev/null
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java
@@ -0,0 +1,40 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator;
+import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
+import net.sourceforge.pmd.util.document.Chars;
+import net.sourceforge.pmd.util.document.TextDocument;
+
+/**
+ *
+ */
+class CppBlockSkipper extends EscapeTranslator {
+
+ private final Pattern skipStart;
+ private final Pattern skipEnd;
+
+ public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) {
+ super(original);
+ skipStart = Pattern.compile("^" + Pattern.quote(skipStartMarker));
+ skipEnd = Pattern.compile("^" + Pattern.quote(skipEndMarker));
+ }
+
+ @Override
+ protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException {
+ Matcher start = skipStart.matcher(input).region(this.bufpos, maxOff);
+ if (start.find()) {
+ Matcher end = skipStart.matcher(input).region(start.end(), maxOff);
+ if (end.find()) {
+ return recordEscape(start.start(), end.end(), Chars.EMPTY);
+ }
+ }
+ return super.gobbleMaxWithoutEscape(maxOff);
+ }
+}
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java
similarity index 96%
rename from pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java
rename to pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java
index 10ffd18503..5a4f0257aa 100644
--- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java
@@ -2,7 +2,7 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
-package net.sourceforge.pmd.lang.cpp.ast;
+package net.sourceforge.pmd.cpd;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeTranslator;
import net.sourceforge.pmd.util.document.Chars;
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
index a9daf1c1e1..284bd05af5 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
@@ -44,8 +44,8 @@ public class JavaTokenizer extends JavaCCTokenizer {
}
@Override
- protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
- return InternalApiBridge.javaTokenDoc(textDoc);
+ protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() {
+ return InternalApiBridge.javaTokenDoc();
}
@Override
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java
index e8921d1ac1..840e06c235 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java
@@ -182,7 +182,7 @@ public final class InternalApiBridge {
((AbstractAnyTypeDeclaration) declaration).setBinaryName(binaryName, canon);
}
- public static JavaccTokenDocument javaTokenDoc(TextDocument fullText) {
- return new JavaTokenDocument(fullText);
+ public static JavaccTokenDocument.TokenDocumentBehavior javaTokenDoc() {
+ return JavaTokenDocument.INSTANCE;
}
}
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java
index fc3e890532..8556ba4094 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java
@@ -5,11 +5,10 @@
package net.sourceforge.pmd.lang.java.ast;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker;
-import net.sourceforge.pmd.util.document.TextDocument;
/**
* Adapter for the JavaParser, using the specified grammar version.
@@ -27,8 +26,8 @@ public class JavaParser extends JjtreeParserAdapter {
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument textDocument) {
- return new JavaTokenDocument(textDocument);
+ protected TokenDocumentBehavior tokenBehavior() {
+ return JavaTokenDocument.INSTANCE;
}
@Override
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java
index d0422442ea..734da5a486 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java
@@ -17,6 +17,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.util.document.TextDocument;
@@ -24,12 +25,15 @@ import net.sourceforge.pmd.util.document.TextDocument;
/**
* {@link JavaccTokenDocument} for Java.
*/
-final class JavaTokenDocument extends JavaccTokenDocument {
+final class JavaTokenDocument extends JavaccTokenDocument.TokenDocumentBehavior {
- JavaTokenDocument(TextDocument fullText) {
- super(fullText);
+ static final JavaTokenDocument INSTANCE = new JavaTokenDocument();
+
+ private JavaTokenDocument() {
+ super(JavaTokenKinds.TOKEN_NAMES, EscapeTranslator.translatorFor(JavaEscapeTranslator::new));
}
+
/**
* Returns true if the given token is a Java comment.
*/
@@ -52,18 +56,14 @@ final class JavaTokenDocument extends JavaccTokenDocument {
}
}
+
@Override
protected boolean isImagePooled(JavaccToken t) {
return t.kind == IDENTIFIER;
}
@Override
- protected @Nullable String describeKindImpl(int kind) {
- return JavaTokenKinds.describe(kind);
- }
-
- @Override
- public JavaccToken createToken(int kind, CharStream jcs, @Nullable String image) {
+ public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream jcs, @Nullable String image) {
switch (kind) {
case RUNSIGNEDSHIFT:
case RSIGNEDSHIFT:
@@ -77,7 +77,7 @@ final class JavaTokenDocument extends JavaccTokenDocument {
jcs.getTokenDocument()
);
default:
- return super.createToken(kind, jcs, image);
+ return super.createToken(self, kind, jcs, image);
}
}
diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
index f2c5f99e1c..73b4f1fc35 100644
--- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
+++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
@@ -11,6 +11,7 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
+import net.sourceforge.pmd.lang.jsp.ast.JspParser;
import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds;
import net.sourceforge.pmd.util.document.TextDocument;
@@ -22,15 +23,8 @@ public class JSPTokenizer extends JavaCCTokenizer {
}
@Override
- protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
- return new JavaccTokenDocument(textDoc) {
- @Override
- protected TextDocument translate(TextDocument text) throws MalformedSourceException {
- try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) {
- return translator.translateDocument();
- }
- }
- };
+ protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() {
+ return JspParser.getTokenBehavior();
}
}
diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java
index 4252bf16d4..87cb1dcd6e 100644
--- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java
+++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java
@@ -4,27 +4,22 @@
package net.sourceforge.pmd.lang.jsp.ast;
-import org.checkerframework.checker.nullness.qual.Nullable;
-
+import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.util.document.TextDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
/**
* JSP language parser.
*/
public final class JspParser extends JjtreeParserAdapter {
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(JspTokenKinds.TOKEN_NAMES);
+
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) {
- return new JavaccTokenDocument(fullText) {
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return JspTokenKinds.describe(kind);
- }
- };
+ protected TokenDocumentBehavior tokenBehavior() {
+ return TOKEN_BEHAVIOR;
}
@Override
@@ -32,4 +27,8 @@ public final class JspParser extends JjtreeParserAdapter {
return new JspParserImpl(cs).CompilationUnit().addTaskInfo(task);
}
+ @InternalApi
+ public static TokenDocumentBehavior getTokenBehavior() {
+ return TOKEN_BEHAVIOR;
+ }
}
diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java
index 058941d176..c4163c0bf0 100644
--- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java
+++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java
@@ -4,18 +4,21 @@
package net.sourceforge.pmd.lang.modelica.ast;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
-import net.sourceforge.pmd.util.document.TextDocument;
public class ModelicaParser extends JjtreeParserAdapter {
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(ModelicaTokenKinds.TOKEN_NAMES);
+
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument textDocument) {
- return new ModelicaTokenDocument(textDocument);
+ protected TokenDocumentBehavior tokenBehavior() {
+ return TOKEN_BEHAVIOR;
}
@Override
diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java
deleted file mode 100644
index 04afae82b5..0000000000
--- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.modelica.ast;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
-import net.sourceforge.pmd.util.document.TextDocument;
-
-
-public class ModelicaTokenDocument extends JavaccTokenDocument {
-
- public ModelicaTokenDocument(TextDocument textDocument) {
- super(textDocument);
- }
-
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return ModelicaTokenKinds.describe(kind);
- }
-}
diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java
index e2beed5425..6b7fc86d69 100644
--- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java
+++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java
@@ -8,20 +8,17 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.util.document.TextDocument;
public class PLSQLParser extends JjtreeParserAdapter {
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(PLSQLTokenKinds.TOKEN_NAMES);
+
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) {
- return new JavaccTokenDocument(fullText) {
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return PLSQLTokenKinds.describe(kind);
- }
- };
+ protected TokenDocumentBehavior tokenBehavior() {
+ return TOKEN_BEHAVIOR;
}
@Override
diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
index 70755980ea..bbd0f3c8bf 100644
--- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
+++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
@@ -13,6 +13,7 @@ import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds;
import net.sourceforge.pmd.util.document.TextDocument;
@@ -23,27 +24,16 @@ public class PythonTokenizer extends JavaCCTokenizer {
private static final Pattern STRING_NL_ESCAPE = Pattern.compile("\\\\\\r?\\n");
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(PythonTokenKinds.TOKEN_NAMES);
+
@Override
protected TokenManager makeLexerImpl(CharStream sourceCode) {
return PythonTokenKinds.newTokenManager(sourceCode);
}
@Override
- protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
- return new PythonTokenDocument(textDoc);
- }
-
- private static class PythonTokenDocument extends JavaccTokenDocument {
-
- PythonTokenDocument(TextDocument fullText) {
- super(fullText);
- }
-
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return PythonTokenKinds.describe(kind);
- }
-
+ protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() {
+ return TOKEN_BEHAVIOR;
}
@Override
diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java
index 95915d6cd0..dfe4665c6a 100644
--- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java
+++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java
@@ -9,10 +9,10 @@ import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
+import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator;
import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator;
-import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException;
import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds;
-import net.sourceforge.pmd.util.document.TextDocument;
/**
* @author sergey.gorbaty
@@ -25,16 +25,9 @@ public class VfTokenizer extends JavaCCTokenizer {
}
@Override
- protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
- return new JavaccTokenDocument(textDoc) {
-
- @Override
- protected TextDocument translate(TextDocument text) throws MalformedSourceException {
- try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) {
- return translator.translateDocument();
- }
- }
- };
+ protected TokenDocumentBehavior newTokenDoc() {
+ return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES,
+ EscapeTranslator.translatorFor(JavaEscapeTranslator::new));
}
}
diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java
index b740cc8c63..07b02632dc 100644
--- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java
+++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java
@@ -4,27 +4,21 @@
package net.sourceforge.pmd.lang.vf.ast;
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
-import net.sourceforge.pmd.util.document.TextDocument;
/**
* Parser for the VisualForce language.
*/
public final class VfParser extends JjtreeParserAdapter {
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES);
+
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) {
- return new JavaccTokenDocument(fullText) {
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return VfTokenKinds.describe(kind);
- }
- };
+ protected TokenDocumentBehavior tokenBehavior() {
+ return TOKEN_BEHAVIOR;
}
@Override
diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java
index 9826dc367c..97f3065fb1 100644
--- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java
+++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java
@@ -7,20 +7,39 @@ package net.sourceforge.pmd.lang.vm.ast;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.ParseException;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter;
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.util.document.TextDocument;
/**
* Adapter for the VmParser.
*/
public class VmParser extends JjtreeParserAdapter {
+ private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(VmTokenKinds.TOKEN_NAMES) {
+
+ @Override
+ public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream cs, @Nullable String image) {
+ String realImage = image == null ? cs.getTokenImage() : image;
+ if (kind == VmTokenKinds.ESCAPE_DIRECTIVE) {
+ realImage = escapedDirective(realImage);
+ }
+
+ return super.createToken(self, kind, cs, realImage);
+ }
+
+ private String escapedDirective(String strImage) {
+ int iLast = strImage.lastIndexOf("\\");
+ String strDirective = strImage.substring(iLast + 1);
+ return strImage.substring(0, iLast / 2) + strDirective;
+ }
+ };
+
@Override
- protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) {
- return new VmTokenDocument(fullText);
+ protected TokenDocumentBehavior tokenBehavior() {
+ return TOKEN_BEHAVIOR;
}
@Override
@@ -29,33 +48,4 @@ public class VmParser extends JjtreeParserAdapter {
}
- private static class VmTokenDocument extends JavaccTokenDocument {
-
- VmTokenDocument(TextDocument fullText) {
- super(fullText);
- }
-
- @Override
- protected @Nullable String describeKindImpl(int kind) {
- return VmTokenKinds.describe(kind);
- }
-
- @Override
- public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) {
- String realImage = image == null ? cs.getTokenImage() : image;
- if (kind == VmTokenKinds.ESCAPE_DIRECTIVE) {
- realImage = escapedDirective(realImage);
- }
-
- return super.createToken(kind, cs, realImage);
- }
-
- private String escapedDirective(String strImage) {
- int iLast = strImage.lastIndexOf("\\");
- String strDirective = strImage.substring(iLast + 1);
- return strImage.substring(0, iLast / 2) + strDirective;
- }
-
- }
-
}