From 551620bbad0ebe87c673ee584bca2e1f7a8b8658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 7 May 2020 17:15:15 +0200 Subject: [PATCH 1/6] Checkout TokenMgrError from 7.0.x, make binary compatible with master --- .../sourceforge/pmd/cpd/ApexTokenizer.java | 2 +- pmd-core/src/main/ant/alljavacc.xml | 20 +-- .../sourceforge/pmd/cpd/AntlrTokenizer.java | 4 +- .../pmd/lang/antlr/AntlrTokenManager.java | 5 + .../pmd/lang/ast/TokenMgrError.java | 138 ++++++++++++++++++ .../net/sourceforge/pmd/util/StringUtil.java | 48 ++++++ .../sourceforge/pmd/cpd/GroovyTokenizer.java | 4 +- 7 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java index bb5f9b6cf1..08927b2635 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java @@ -42,7 +42,7 @@ public class ApexTokenizer implements Tokenizer { ApexLexer lexer = new ApexLexer(ass) { @Override public void emitErrorMessage(String msg) { - throw new TokenMgrError(msg, TokenMgrError.LEXICAL_ERROR); + throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null); } }; diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml index 8c0261cb4a..5d00c63531 100644 --- a/pmd-core/src/main/ant/alljavacc.xml +++ b/pmd-core/src/main/ant/alljavacc.xml @@ -63,6 +63,10 @@ + @@ -84,22 +88,6 @@ - - - - - - - - diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index 05b18eea5b..074ced829e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -41,9 +41,7 @@ public abstract class AntlrTokenizer implements Tokenizer { } catch (final AntlrTokenManager.ANTLRSyntaxError err) { // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line option - throw new TokenMgrError("Lexical error in file " + tokenManager.getFileName() + " at line " - + err.getLine() + ", column " + err.getColumn() + ". Encountered: " + err.getMessage(), - TokenMgrError.LEXICAL_ERROR); + throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), err.getCause()); } finally { tokenEntries.add(TokenEntry.getEOF()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java index 48434eca01..b47ed0818c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java @@ -11,6 +11,7 @@ import org.antlr.v4.runtime.Recognizer; import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.TokenMgrError; /** * Generic token manager implementation for all Antlr lexers. @@ -72,6 +73,10 @@ public class AntlrTokenManager implements TokenManager { } } + /** + * @deprecated On 7.0.x branch this has been replaced by {@link TokenMgrError} already + */ + @Deprecated public static class ANTLRSyntaxError extends RuntimeException { private static final long serialVersionUID = 1L; private final int line; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java new file mode 100644 index 0000000000..985f79f4e1 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java @@ -0,0 +1,138 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast; + +import net.sourceforge.pmd.annotation.InternalApi; +import net.sourceforge.pmd.util.StringUtil; + +/** + * An error thrown during lexical analysis of a file. + */ +public final class TokenMgrError extends RuntimeException { + + private final int line; + private final int column; + private final String filename; + + // these constants are deprecated because they're useless, + // they've been removed from 7.0.x + + @Deprecated + public static final int LEXICAL_ERROR = 0; + @Deprecated + public static final int STATIC_LEXER_ERROR = 1; + @Deprecated + public static final int INVALID_LEXICAL_STATE = 2; + @Deprecated + public static final int LOOP_DETECTED = 3; + + /** + * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)} + */ + @Deprecated + public TokenMgrError() { + this("NO_MESSAGE", LEXICAL_ERROR); + } + + /** + * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)} + */ + @Deprecated + public TokenMgrError(String message, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) { + super(message); + this.line = -1; + this.column = -1; + this.filename = null; + } + + /** + * Create a new exception. + * + * @param line Line number + * @param column Column number + * @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later + * @param message Message of the error + * @param cause Cause of the error, if any + */ + public TokenMgrError(int line, int column, /*@Nullable*/ String filename, String message, /*@Nullable*/ Throwable cause) { + super(message, cause); + this.line = line; + this.column = column; + this.filename = filename; + } + + /** + * Constructor called by JavaCC. + * + * @deprecated This should only be used by the Javacc implementations we maintain, will change in 7.0 + */ + @InternalApi + @Deprecated + public TokenMgrError(boolean eofSeen, int lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) { + super(makeReason(eofSeen, lexStateName, errorAfter, curChar)); + line = errorLine; + column = errorColumn; + filename = null; // may be replaced with #withFileName + } + + public int getLine() { + return line; + } + + public int getColumn() { + return column; + } + + public /*@Nullable*/ String getFilename() { + return filename; + } + + /** + * @deprecated Use {@link StringUtil#escapeJava(String)} + */ + @Deprecated + protected static String addEscapes(String str) { + return StringUtil.escapeJava(str); + } + + + @Deprecated + protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { // SUPPRESS CHECKSTYLE yes it's ugly, but it's for compatibility + return makeMessage(null, errorLine, errorColumn, makeReason(eofSeen, lexState, errorAfter, curChar)); + } + + @Override + public String getMessage() { + return makeMessage(filename, line, column, super.getMessage()); + } + + private static String makeMessage(String filename, int line, int column, String message) { + String leader = filename != null ? "Lexical error in file " + filename : "Lexical error"; + return leader + " at line " + line + ", column " + column + ". Encountered: " + message; + } + + /** + * Replace the file name of this error. + * + * @param filename New filename + * + * @return A new exception + */ + public TokenMgrError withFileName(String filename) { + return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause()); + } + + private static String makeReason(boolean eofseen, int lexStateName, String errorAfter, char curChar) { + String message; + if (eofseen) { + message = " "; + } else { + message = "\"" + StringUtil.escapeJava(String.valueOf(curChar)) + "\"" + " (" + (int) curChar + "), "; + } + message += "after : \"" + StringUtil.escapeJava(errorAfter) + "\" (in lexical state " + lexStateName + ")"; + + return message; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java index d24165fa0a..fcb8aaaab8 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java @@ -659,4 +659,52 @@ public final class StringUtil { return sb.toString(); } + + /** + * Replaces unprintable characters by their escaped (or unicode escaped) + * equivalents in the given string + */ + public static String escapeJava(String str) { + StringBuilder retval = new StringBuilder(); + for (int i = 0; i < str.length(); i++) { + final char ch = str.charAt(i); + switch (ch) { + case 0: + break; + case '\b': + retval.append("\\b"); + break; + case '\t': + retval.append("\\t"); + break; + case '\n': + retval.append("\\n"); + break; + case '\f': + retval.append("\\f"); + break; + case '\r': + retval.append("\\r"); + break; + case '\"': + retval.append("\\\""); + break; + case '\'': + retval.append("\\'"); + break; + case '\\': + retval.append("\\\\"); + break; + default: + if (ch < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u").append(s.substring(s.length() - 4)); + } else { + retval.append(ch); + } + break; + } + } + return retval.toString(); + } } diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java index 7470ad8ee9..e9ce320a0f 100644 --- a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java +++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java @@ -42,9 +42,7 @@ public class GroovyTokenizer implements Tokenizer { // they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line // option - throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine() - + ", column " + lexer.getColumn() + ". Encountered: " + err.getMessage(), - TokenMgrError.LEXICAL_ERROR); + throw new TokenMgrError(lexer.getLine(), lexer.getColumn(), sourceCode.getFileName(), err.getMessage(), err); } finally { tokenEntries.add(TokenEntry.getEOF()); } From f8b8c229f89b91a644017e7704b22abdbbbdacfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 7 May 2020 17:29:49 +0200 Subject: [PATCH 2/6] Make JavaCharStream not throw Error --- pmd-core/src/main/ant/alljavacc.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml index 5d00c63531..9fa656505f 100644 --- a/pmd-core/src/main/ant/alljavacc.xml +++ b/pmd-core/src/main/ant/alljavacc.xml @@ -63,10 +63,14 @@ + + + + + token="throw new Error" + value="throw new RuntimeException" /> + From eeed02ab86eb571339dc44ed9f27c79591a7617e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 7 May 2020 17:40:59 +0200 Subject: [PATCH 3/6] Fix tests --- .../src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java | 2 +- pmd-core/src/main/ant/alljavacc.xml | 5 +++++ .../net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java index 08927b2635..bb5f9b6cf1 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java @@ -42,7 +42,7 @@ public class ApexTokenizer implements Tokenizer { ApexLexer lexer = new ApexLexer(ass) { @Override public void emitErrorMessage(String msg) { - throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null); + throw new TokenMgrError(msg, TokenMgrError.LEXICAL_ERROR); } }; diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml index 9fa656505f..2389515967 100644 --- a/pmd-core/src/main/ant/alljavacc.xml +++ b/pmd-core/src/main/ant/alljavacc.xml @@ -89,6 +89,11 @@ + + + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 2721f5cbc6..512e1ff836 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -14,6 +14,7 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.ast.TokenMgrError; public abstract class JavaCCTokenizer implements Tokenizer { @@ -39,6 +40,8 @@ public abstract class JavaCCTokenizer implements Tokenizer { tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName())); currentToken = tokenFilter.getNextToken(); } + } catch (TokenMgrError e) { + throw e.withFileName(sourceCode.getFileName()); } finally { tokenEntries.add(TokenEntry.getEOF()); } From bf027382f591673afffdf47437f0d8ecd1921d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 7 May 2020 18:05:24 +0200 Subject: [PATCH 4/6] Test --- pmd-core/src/main/ant/alljavacc.xml | 10 ++++++---- .../pmd/lang/java/ast/ParserCornersTest.java | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml index 2389515967..e792989ba3 100644 --- a/pmd-core/src/main/ant/alljavacc.xml +++ b/pmd-core/src/main/ant/alljavacc.xml @@ -67,9 +67,10 @@ + + token="throw new Error(t.getMessage())" + value="throw t" /> + + token="throw new Error(t.getMessage())" + value="throw t" /> Date: Thu, 7 May 2020 18:19:50 +0200 Subject: [PATCH 5/6] Update release notes --- docs/pages/release_notes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 20bacf1624..081af7ba45 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -18,6 +18,12 @@ This is a {{ site.pmd.release_type }} release. ### API Changes +#### Deprecated APIs + +* Some members of {% jdoc core::lang.ast.TokenMgrError %}, in particular, a new constructor is available +that should be preferred to the old ones +* {% jdoc core::lang.antlr.AntlrTokenManager.ANTLRSyntaxError %} + ### External Contributions * [#2446](https://github.com/pmd/pmd/pull/2446): \[core] Update maven-compiler-plugin to 3.8.1 - [Artem Krosheninnikov](https://github.com/KroArtem) From ca341e85d4df9b03ce7564fd1513aa283d9d7367 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Mon, 18 May 2020 20:27:03 +0200 Subject: [PATCH 6/6] [doc] Update release notes, fixes #2472 --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 081af7ba45..f455754d7d 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,6 +16,9 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java + * [#2472](https://github.com/pmd/pmd/issues/2472): \[java] JavaCharStream throws an Error on invalid escape + ### API Changes #### Deprecated APIs