Merge branch 'pr-2474'

[core] Stop JavaCharStream from throwing Error #2474

docs/pages/release_notes.md

@@ -38,6 +38,8 @@ This is useful to find duplicated sections in XML files.
     *   [#2468](https://github.com/pmd/pmd/issues/2468): \[apex] Unused Local Variable fails on blocks
 *   core
     *   [#2484](https://github.com/pmd/pmd/issues/2484): \[core] Update maven-enforcer-plugin to require Java 118
+*   java
+    *   [#2472](https://github.com/pmd/pmd/issues/2472): \[java] JavaCharStream throws an Error on invalid escape
 *   java-bestpractices
     *   [#2288](https://github.com/pmd/pmd/issues/2288): \[java] JUnitTestsShouldIncludeAssert: Add support for Hamcrest MatcherAssert.assertThat
 *   java-errorprone
@@ -49,7 +51,10 @@ This is useful to find duplicated sections in XML files.
 
 #### Deprecated APIs
 
-* {% jdoc !ca!core::lang.BaseLanguageModule#addVersion(String, LanguageVersionHandler, boolean) %}
+*   {% jdoc !ca!core::lang.BaseLanguageModule#addVersion(String, LanguageVersionHandler, boolean) %}
+*   Some members of {% jdoc core::lang.ast.TokenMgrError %}, in particular, a new constructor is available
+    that should be preferred to the old ones
+*   {% jdoc core::lang.antlr.AntlrTokenManager.ANTLRSyntaxError %}
 
 #### Experimental APIs
 
@@ -59,8 +64,6 @@ see its javadoc for details
 * The experimental methods in {% jdoc !ca!core::lang.BaseLanguageModule %} have been replaced by a
 definitive API.
 
-
-
 ### External Contributions
 
 *   [#2446](https://github.com/pmd/pmd/pull/2446): \[core] Update maven-compiler-plugin to 3.8.1 - [Artem Krosheninnikov](https://github.com/KroArtem)

pmd-core/src/main/ant/alljavacc.xml

@@ -63,6 +63,15 @@
         <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/JavaCharStream.java"
                  token="class JavaCharStream"
                  value="@Deprecated @net.sourceforge.pmd.annotation.InternalApi class JavaCharStream implements CharStream" />
+        <replaceregexp file="${target}/net/sourceforge/pmd/lang/ast/dummy/JavaCharStream.java">
+            <regexp pattern='throw new Error\("Invalid escape character at line " \+ line \+\s+" column " \+ column \+ "."\);' />
+            <substitution expression='throw new TokenMgrError(line, column, AbstractTokenManager.getFileName(), "Invalid unicode escape", null);' />
+        </replaceregexp>
+        <!-- This is in ExpandBuf, the exception this may throw is OutOfMemoryError, which is stupidly erased by javacc -->
+        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/JavaCharStream.java"
+                 token="throw new Error(t.getMessage())"
+                 value="throw t" />
+
         <move overwrite="true"
               file="${target}/net/sourceforge/pmd/lang/ast/dummy/JavaCharStream.java"
               tofile="${target}/net/sourceforge/pmd/lang/ast/JavaCharStream.java" />
@@ -81,25 +90,15 @@
         <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/SimpleCharStream.java"
                  token="public class SimpleCharStream"
                  value="@Deprecated @net.sourceforge.pmd.annotation.InternalApi public class SimpleCharStream implements CharStream" />
+        <!-- This is in ExpandBuf, the exception this may throw is OutOfMemoryError, which is stupidly erased by javacc -->
+        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/SimpleCharStream.java"
+                 token="throw new Error(t.getMessage())"
+                 value="throw t" />
+
+
         <move overwrite="true"
               file="${target}/net/sourceforge/pmd/lang/ast/dummy/SimpleCharStream.java"
               tofile="${target}/net/sourceforge/pmd/lang/ast/SimpleCharStream.java" />
-        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/TokenMgrError.java"
-                 token="net.sourceforge.pmd.lang.ast.dummy"
-                 value="net.sourceforge.pmd.lang.ast" />
-        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/TokenMgrError.java"
-                 token="extends Error"
-                 value="extends RuntimeException" />
-        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/TokenMgrError.java"
-                 token="static final int"
-                 value="public static final int" />
-        <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/TokenMgrError.java">
-            <replacetoken><![CDATA["Lexical error at line "]]></replacetoken>
-            <replacevalue><![CDATA["Lexical error in file " + AbstractTokenManager.getFileName() + " at line "]]></replacevalue>
-        </replace>
-        <move overwrite="true"
-              file="${target}/net/sourceforge/pmd/lang/ast/dummy/TokenMgrError.java"
-              tofile="${target}/net/sourceforge/pmd/lang/ast/TokenMgrError.java" />
 
         <delete dir="${target}/net/sourceforge/pmd/lang/ast/dummy" />
 

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java

@@ -41,9 +41,7 @@ public abstract class AntlrTokenizer implements Tokenizer {
         } catch (final AntlrTokenManager.ANTLRSyntaxError err) {
             // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
             // when CPD is executed with the '--skipLexicalErrors' command line option
-            throw new TokenMgrError("Lexical error in file " + tokenManager.getFileName() + " at line "
-                    + err.getLine() + ", column " + err.getColumn() + ".  Encountered: " + err.getMessage(),
-                    TokenMgrError.LEXICAL_ERROR);
+            throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), err.getCause());
         } finally {
             tokenEntries.add(TokenEntry.getEOF());
         }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java

@@ -14,6 +14,7 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.GenericToken;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 public abstract class JavaCCTokenizer implements Tokenizer {
 
@@ -39,6 +40,8 @@ public abstract class JavaCCTokenizer implements Tokenizer {
                 tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName()));
                 currentToken = tokenFilter.getNextToken();
             }
+        } catch (TokenMgrError e) {
+            throw e.withFileName(sourceCode.getFileName());
         } finally {
             tokenEntries.add(TokenEntry.getEOF());
         }

pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java

@@ -11,6 +11,7 @@ import org.antlr.v4.runtime.Recognizer;
 
 import net.sourceforge.pmd.cpd.token.AntlrToken;
 import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 /**
  * Generic token manager implementation for all Antlr lexers.
@@ -72,6 +73,10 @@ public class AntlrTokenManager implements TokenManager {
         }
     }
 
+    /**
+     * @deprecated On 7.0.x branch this has been replaced by {@link TokenMgrError} already
+     */
+    @Deprecated
     public static class ANTLRSyntaxError extends RuntimeException {
         private static final long serialVersionUID = 1L;
         private final int line;

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java

@@ -0,0 +1,138 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ast;
+
+import net.sourceforge.pmd.annotation.InternalApi;
+import net.sourceforge.pmd.util.StringUtil;
+
+/**
+ * An error thrown during lexical analysis of a file.
+ */
+public final class TokenMgrError extends RuntimeException {
+
+    private final int line;
+    private final int column;
+    private final String filename;
+
+    // these constants are deprecated because they're useless,
+    // they've been removed from 7.0.x
+
+    @Deprecated
+    public static final int LEXICAL_ERROR = 0;
+    @Deprecated
+    public static final int STATIC_LEXER_ERROR = 1;
+    @Deprecated
+    public static final int INVALID_LEXICAL_STATE = 2;
+    @Deprecated
+    public static final int LOOP_DETECTED = 3;
+
+    /**
+     * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)}
+     */
+    @Deprecated
+    public TokenMgrError() {
+        this("NO_MESSAGE", LEXICAL_ERROR);
+    }
+
+    /**
+     * @deprecated Use {@link #TokenMgrError(int, int, String, String, Throwable)}
+     */
+    @Deprecated
+    public TokenMgrError(String message, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) {
+        super(message);
+        this.line = -1;
+        this.column = -1;
+        this.filename = null;
+    }
+
+    /**
+     * Create a new exception.
+     *
+     * @param line     Line number
+     * @param column   Column number
+     * @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later
+     * @param message  Message of the error
+     * @param cause    Cause of the error, if any
+     */
+    public TokenMgrError(int line, int column, /*@Nullable*/ String filename, String message, /*@Nullable*/ Throwable cause) {
+        super(message, cause);
+        this.line = line;
+        this.column = column;
+        this.filename = filename;
+    }
+
+    /**
+     * Constructor called by JavaCC.
+     *
+     * @deprecated This should only be used by the Javacc implementations we maintain, will change in 7.0
+     */
+    @InternalApi
+    @Deprecated
+    public TokenMgrError(boolean eofSeen, int lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar, @SuppressWarnings("PMD.UnusedFormalParameter") int reason) {
+        super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
+        line = errorLine;
+        column = errorColumn;
+        filename = null; // may be replaced with #withFileName
+    }
+
+    public int getLine() {
+        return line;
+    }
+
+    public int getColumn() {
+        return column;
+    }
+
+    public /*@Nullable*/ String getFilename() {
+        return filename;
+    }
+
+    /**
+     * @deprecated Use {@link StringUtil#escapeJava(String)}
+     */
+    @Deprecated
+    protected static String addEscapes(String str) {
+        return StringUtil.escapeJava(str);
+    }
+
+
+    @Deprecated
+    protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { // SUPPRESS CHECKSTYLE yes it's ugly, but it's for compatibility
+        return makeMessage(null, errorLine, errorColumn, makeReason(eofSeen, lexState, errorAfter, curChar));
+    }
+
+    @Override
+    public String getMessage() {
+        return makeMessage(filename, line, column, super.getMessage());
+    }
+
+    private static String makeMessage(String filename, int line, int column, String message) {
+        String leader = filename != null ? "Lexical error in file " + filename : "Lexical error";
+        return leader + " at line " + line + ", column " + column + ".  Encountered: " + message;
+    }
+
+    /**
+     * Replace the file name of this error.
+     *
+     * @param filename New filename
+     *
+     * @return A new exception
+     */
+    public TokenMgrError withFileName(String filename) {
+        return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause());
+    }
+
+    private static String makeReason(boolean eofseen, int lexStateName, String errorAfter, char curChar) {
+        String message;
+        if (eofseen) {
+            message = "<EOF> ";
+        } else {
+            message = "\"" + StringUtil.escapeJava(String.valueOf(curChar)) + "\"" + " (" + (int) curChar + "), ";
+        }
+        message += "after : \"" + StringUtil.escapeJava(errorAfter) + "\" (in lexical state " + lexStateName + ")";
+
+        return message;
+    }
+}

pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java

@@ -659,4 +659,52 @@ public final class StringUtil {
         return sb.toString();
     }
 
+
+    /**
+     * Replaces unprintable characters by their escaped (or unicode escaped)
+     * equivalents in the given string
+     */
+    public static String escapeJava(String str) {
+        StringBuilder retval = new StringBuilder();
+        for (int i = 0; i < str.length(); i++) {
+            final char ch = str.charAt(i);
+            switch (ch) {
+            case 0:
+                break;
+            case '\b':
+                retval.append("\\b");
+                break;
+            case '\t':
+                retval.append("\\t");
+                break;
+            case '\n':
+                retval.append("\\n");
+                break;
+            case '\f':
+                retval.append("\\f");
+                break;
+            case '\r':
+                retval.append("\\r");
+                break;
+            case '\"':
+                retval.append("\\\"");
+                break;
+            case '\'':
+                retval.append("\\'");
+                break;
+            case '\\':
+                retval.append("\\\\");
+                break;
+            default:
+                if (ch < 0x20 || ch > 0x7e) {
+                    String s = "0000" + Integer.toString(ch, 16);
+                    retval.append("\\u").append(s.substring(s.length() - 4));
+                } else {
+                    retval.append(ch);
+                }
+                break;
+            }
+        }
+        return retval.toString();
+    }
 }

pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java

@@ -42,9 +42,7 @@ public class GroovyTokenizer implements Tokenizer {
             // they are correctly handled
             // when CPD is executed with the '--skipLexicalErrors' command line
             // option
-            throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine()
-                    + ", column " + lexer.getColumn() + ".  Encountered: " + err.getMessage(),
-                    TokenMgrError.LEXICAL_ERROR);
+            throw new TokenMgrError(lexer.getLine(), lexer.getColumn(), sourceCode.getFileName(), err.getMessage(), err);
         } finally {
             tokenEntries.add(TokenEntry.getEOF());
         }

pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java

@@ -15,6 +15,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.java.JavaParsingHelper;
 import net.sourceforge.pmd.lang.symboltable.NameDeclaration;
 import net.sourceforge.pmd.lang.symboltable.NameOccurrence;
@@ -31,6 +32,14 @@ public class ParserCornersTest {
     @Rule
     public ExpectedException expect = ExpectedException.none();
 
+
+    @Test
+    public void testInvalidUnicodeEscape() {
+        expect.expect(TokenMgrError.class); // previously Error
+        expect.expectMessage("Lexical error in file (no file name provided) at line 1, column 2.  Encountered: Invalid unicode escape");
+        java.parse("\\u00k0");
+    }
+
     /**
      * #1107 PMD 5.0.4 couldn't parse call of parent outer java class method
      * from inner class.