diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index ba7d7fd760..39d52cde97 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -38,6 +38,8 @@ This is useful to find duplicated sections in XML files.
* [#2468](https://github.com/pmd/pmd/issues/2468): \[apex] Unused Local Variable fails on blocks
* core
* [#2484](https://github.com/pmd/pmd/issues/2484): \[core] Update maven-enforcer-plugin to require Java 118
+* java
+ * [#2472](https://github.com/pmd/pmd/issues/2472): \[java] JavaCharStream throws an Error on invalid escape
* java-bestpractices
* [#2288](https://github.com/pmd/pmd/issues/2288): \[java] JUnitTestsShouldIncludeAssert: Add support for Hamcrest MatcherAssert.assertThat
* java-errorprone
@@ -49,7 +51,10 @@ This is useful to find duplicated sections in XML files.
#### Deprecated APIs
-* {% jdoc !ca!core::lang.BaseLanguageModule#addVersion(String, LanguageVersionHandler, boolean) %}
+* {% jdoc !ca!core::lang.BaseLanguageModule#addVersion(String, LanguageVersionHandler, boolean) %}
+* Some members of {% jdoc core::lang.ast.TokenMgrError %}, in particular, a new constructor is available
+ that should be preferred to the old ones
+* {% jdoc core::lang.antlr.AntlrTokenManager.ANTLRSyntaxError %}
#### Experimental APIs
@@ -59,8 +64,6 @@ see its javadoc for details
* The experimental methods in {% jdoc !ca!core::lang.BaseLanguageModule %} have been replaced by a
definitive API.
-
-
### External Contributions
* [#2446](https://github.com/pmd/pmd/pull/2446): \[core] Update maven-compiler-plugin to 3.8.1 - [Artem Krosheninnikov](https://github.com/KroArtem)
diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml
index 8c0261cb4a..e792989ba3 100644
--- a/pmd-core/src/main/ant/alljavacc.xml
+++ b/pmd-core/src/main/ant/alljavacc.xml
@@ -63,6 +63,15 @@
+
+
+
+
+
+
+
@@ -81,25 +90,15 @@
+
+
+
+
-
-
-
-
-
-
-
-
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
index 79f70f7db6..4bc13d81fa 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
@@ -41,9 +41,7 @@ public abstract class AntlrTokenizer implements Tokenizer {
} catch (final AntlrTokenManager.ANTLRSyntaxError err) {
// Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
// when CPD is executed with the '--skipLexicalErrors' command line option
- throw new TokenMgrError("Lexical error in file " + tokenManager.getFileName() + " at line "
- + err.getLine() + ", column " + err.getColumn() + ". Encountered: " + err.getMessage(),
- TokenMgrError.LEXICAL_ERROR);
+ throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), err.getCause());
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
index 2721f5cbc6..512e1ff836 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
@@ -14,6 +14,7 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
public abstract class JavaCCTokenizer implements Tokenizer {
@@ -39,6 +40,8 @@ public abstract class JavaCCTokenizer implements Tokenizer {
tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName()));
currentToken = tokenFilter.getNextToken();
}
+ } catch (TokenMgrError e) {
+ throw e.withFileName(sourceCode.getFileName());
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
index 48434eca01..b47ed0818c 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
@@ -11,6 +11,7 @@ import org.antlr.v4.runtime.Recognizer;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Generic token manager implementation for all Antlr lexers.
@@ -72,6 +73,10 @@ public class AntlrTokenManager implements TokenManager {
}
}
+ /**
+ * @deprecated On 7.0.x branch this has been replaced by {@link TokenMgrError} already
+ */
+ @Deprecated
public static class ANTLRSyntaxError extends RuntimeException {
private static final long serialVersionUID = 1L;
private final int line;
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java
new file mode 100644
index 0000000000..985f79f4e1
--- /dev/null
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java
@@ -0,0 +1,138 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ast;
+
+import net.sourceforge.pmd.annotation.InternalApi;
+import net.sourceforge.pmd.util.StringUtil;
+
+/**
+ * An error thrown during lexical analysis of a file.
+ */
+public final class TokenMgrError extends RuntimeException {
+
+ private final int line;
+ private final int column;
+ private final String filename;
+
+ // these constants are deprecated because they're useless,
+ // they've been removed from 7.0.x
+
+ @Deprecated
+ public static final int LEXICAL_ERROR = 0;
+ @Deprecated
+ public static final int STATIC_LEXER_ERROR = 1;
+ @Deprecated
+ public static final int INVALID_LEXICAL_STATE = 2;
+ @Deprecated
+ public static final int LOOP_DETECTED = 3;
+
+ /**
+ * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)}
+ */
+ @Deprecated
+ public TokenMgrError() {
+ this("NO_MESSAGE", LEXICAL_ERROR);
+ }
+
+ /**
+ * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)}
+ */
+ @Deprecated
+ public TokenMgrError(String message, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) {
+ super(message);
+ this.line = -1;
+ this.column = -1;
+ this.filename = null;
+ }
+
+ /**
+ * Create a new exception.
+ *
+ * @param line Line number
+ * @param column Column number
+ * @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later
+ * @param message Message of the error
+ * @param cause Cause of the error, if any
+ */
+ public TokenMgrError(int line, int column, /*@Nullable*/ String filename, String message, /*@Nullable*/ Throwable cause) {
+ super(message, cause);
+ this.line = line;
+ this.column = column;
+ this.filename = filename;
+ }
+
+ /**
+ * Constructor called by JavaCC.
+ *
+ * @deprecated This should only be used by the Javacc implementations we maintain, will change in 7.0
+ */
+ @InternalApi
+ @Deprecated
+ public TokenMgrError(boolean eofSeen, int lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) {
+ super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
+ line = errorLine;
+ column = errorColumn;
+ filename = null; // may be replaced with #withFileName
+ }
+
+ public int getLine() {
+ return line;
+ }
+
+ public int getColumn() {
+ return column;
+ }
+
+ public /*@Nullable*/ String getFilename() {
+ return filename;
+ }
+
+ /**
+ * @deprecated Use {@link StringUtil#escapeJava(String)}
+ */
+ @Deprecated
+ protected static String addEscapes(String str) {
+ return StringUtil.escapeJava(str);
+ }
+
+
+ @Deprecated
+ protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { // SUPPRESS CHECKSTYLE yes it's ugly, but it's for compatibility
+ return makeMessage(null, errorLine, errorColumn, makeReason(eofSeen, lexState, errorAfter, curChar));
+ }
+
+ @Override
+ public String getMessage() {
+ return makeMessage(filename, line, column, super.getMessage());
+ }
+
+ private static String makeMessage(String filename, int line, int column, String message) {
+ String leader = filename != null ? "Lexical error in file " + filename : "Lexical error";
+ return leader + " at line " + line + ", column " + column + ". Encountered: " + message;
+ }
+
+ /**
+ * Replace the file name of this error.
+ *
+ * @param filename New filename
+ *
+ * @return A new exception
+ */
+ public TokenMgrError withFileName(String filename) {
+ return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause());
+ }
+
+ private static String makeReason(boolean eofseen, int lexStateName, String errorAfter, char curChar) {
+ String message;
+ if (eofseen) {
+ message = " ";
+ } else {
+ message = "\"" + StringUtil.escapeJava(String.valueOf(curChar)) + "\"" + " (" + (int) curChar + "), ";
+ }
+ message += "after : \"" + StringUtil.escapeJava(errorAfter) + "\" (in lexical state " + lexStateName + ")";
+
+ return message;
+ }
+}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java
index d24165fa0a..fcb8aaaab8 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java
@@ -659,4 +659,52 @@ public final class StringUtil {
return sb.toString();
}
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ public static String escapeJava(String str) {
+ StringBuilder retval = new StringBuilder();
+ for (int i = 0; i < str.length(); i++) {
+ final char ch = str.charAt(i);
+ switch (ch) {
+ case 0:
+ break;
+ case '\b':
+ retval.append("\\b");
+ break;
+ case '\t':
+ retval.append("\\t");
+ break;
+ case '\n':
+ retval.append("\\n");
+ break;
+ case '\f':
+ retval.append("\\f");
+ break;
+ case '\r':
+ retval.append("\\r");
+ break;
+ case '\"':
+ retval.append("\\\"");
+ break;
+ case '\'':
+ retval.append("\\'");
+ break;
+ case '\\':
+ retval.append("\\\\");
+ break;
+ default:
+ if (ch < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u").append(s.substring(s.length() - 4));
+ } else {
+ retval.append(ch);
+ }
+ break;
+ }
+ }
+ return retval.toString();
+ }
}
diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java
index 7470ad8ee9..e9ce320a0f 100644
--- a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java
+++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java
@@ -42,9 +42,7 @@ public class GroovyTokenizer implements Tokenizer {
// they are correctly handled
// when CPD is executed with the '--skipLexicalErrors' command line
// option
- throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine()
- + ", column " + lexer.getColumn() + ". Encountered: " + err.getMessage(),
- TokenMgrError.LEXICAL_ERROR);
+ throw new TokenMgrError(lexer.getLine(), lexer.getColumn(), sourceCode.getFileName(), err.getMessage(), err);
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java
index a710ec3d51..83501273fd 100644
--- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java
+++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java
@@ -15,6 +15,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.java.JavaParsingHelper;
import net.sourceforge.pmd.lang.symboltable.NameDeclaration;
import net.sourceforge.pmd.lang.symboltable.NameOccurrence;
@@ -31,6 +32,14 @@ public class ParserCornersTest {
@Rule
public ExpectedException expect = ExpectedException.none();
+
+ @Test
+ public void testInvalidUnicodeEscape() {
+ expect.expect(TokenMgrError.class); // previously Error
+ expect.expectMessage("Lexical error in file (no file name provided) at line 1, column 2. Encountered: Invalid unicode escape");
+ java.parse("\\u00k0");
+ }
+
/**
* #1107 PMD 5.0.4 couldn't parse call of parent outer java class method
* from inner class.