[core] Rename TokenMgrError to LexException

See #4065

docs/pages/release_notes.md

@@ -159,6 +159,11 @@ The following previously deprecated classes have been removed:
       If the current version is needed, then `Node.getTextDocument().getLanguageVersion()` can be used. This
       is the version that has been selected via CLI `--use-version` parameter.
 
+**Renamed classes**
+
+* pmd-core
+  * {%jdoc_old core::lang.ast.TokenMgrError %} has been renamed to {% jdoc core::lang.ast.LexException %}
+
 #### External Contributions
 * [#4640](https://github.com/pmd/pmd/pull/4640): \[cli] Launch script fails if run via "bash pmd" - [Shai Bennathan](https://github.com/shai-bennathan) (@shai-bennathan)
 * [#4673](https://github.com/pmd/pmd/pull/4673): \[javascript] CPD: Added support for decorator notation - [Wener](https://github.com/wener-tiobe) (@wener-tiobe)

javacc-wrapper.xml

@@ -280,6 +280,13 @@
             <file name="${tokenmgr-file}" />
         </replaceregexp>
 
+        <!-- Use own LexException instead of JavaCC's TokenMgrError -->
+        <replaceregexp>
+            <regexp pattern='throw new TokenMgrError\(EOFSeen' />
+            <substitution expression='throw new net.sourceforge.pmd.lang.ast.LexException(EOFSeen' />
+            <file name="${tokenmgr-file}" />
+        </replaceregexp>
+
         <!-- Useless argument, also replace lex state ID with its name  -->
         <replaceregexp>
             <regexp pattern='curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR\)' />

pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java

@@ -23,7 +23,7 @@ import net.sourceforge.pmd.internal.util.IOUtil;
 import net.sourceforge.pmd.lang.Language;
 import net.sourceforge.pmd.lang.LanguagePropertyBundle;
 import net.sourceforge.pmd.lang.ast.FileAnalysisException;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.document.FileCollector;
 import net.sourceforge.pmd.lang.document.FileId;
 import net.sourceforge.pmd.lang.document.TextDocument;
@@ -137,7 +137,7 @@ public final class CpdAnalysis implements AutoCloseable {
         this.listener = cpdListener;
     }
 
-    private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) throws IOException, TokenMgrError {
+    private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) throws IOException, LexException {
         LOGGER.trace("Tokenizing {}", document.getFileId().getAbsolutePath());
         int lastTokenSize = tokens.size();
         Tokenizer.tokenize(tokenizer, document, tokens);
@@ -170,7 +170,7 @@ public final class CpdAnalysis implements AutoCloseable {
                     int newTokens = doTokenize(textDocument, tokenizers.get(textFile.getLanguageVersion().getLanguage()), tokens);
                     numberOfTokensPerFile.put(textDocument.getFileId(), newTokens);
                     listener.addedFile(1);
-                } catch (TokenMgrError | IOException e) {
+                } catch (LexException | IOException e) {
                     if (e instanceof FileAnalysisException) { // NOPMD
                         ((FileAnalysisException) e).setFileId(textFile.getFileId());
                     }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.document.FileLocation;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
@@ -43,7 +43,7 @@ public interface TokenFactory extends AutoCloseable {
         recordToken(image, location.getStartLine(), location.getStartColumn(), location.getEndLine(), location.getEndColumn());
     }
 
-    TokenMgrError makeLexException(int line, int column, String message, @Nullable Throwable cause);
+    LexException makeLexException(int line, int column, String message, @Nullable Throwable cause);
 
     /**
      * Sets the image of an existing token entry.

pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java

@@ -14,7 +14,7 @@ import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import net.sourceforge.pmd.annotation.InternalApi;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.document.FileId;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
@@ -117,8 +117,8 @@ public class Tokens {
             }
 
             @Override
-            public TokenMgrError makeLexException(int line, int column, String message, @Nullable Throwable cause) {
+            public LexException makeLexException(int line, int column, String message, @Nullable Throwable cause) {
-                return new TokenMgrError(line, column, fileId, message, cause);
+                return new LexException(line, column, fileId, message, cause);
             }
 
             @Override

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java

@@ -16,7 +16,7 @@ import net.sourceforge.pmd.lang.document.FileLocation;
 /**
  * An exception that occurs while processing a file. Subtypes include
  * <ul>
- * <li>{@link TokenMgrError}: lexical syntax errors
+ * <li>{@link LexException}: lexical syntax errors
  * <li>{@link ParseException}: syntax errors
  * <li>{@link SemanticException}: exceptions occurring after the parsing
  * phase, because the source code is semantically invalid

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/LexException.java

@@ -16,8 +16,10 @@ import net.sourceforge.pmd.util.StringUtil;
 
 /**
  * An error thrown during lexical analysis of a file.
+ *
+ * <p>Note: This exception has been called TokenMgrError in PMD 6.</p>
  */
-public final class TokenMgrError extends FileAnalysisException {
+public final class LexException extends FileAnalysisException {
 
     private final int line;
     private final int column;
@@ -31,7 +33,7 @@ public final class TokenMgrError extends FileAnalysisException {
      * @param message  Message of the error
      * @param cause    Cause of the error, if any
      */
-    public TokenMgrError(int line, int column, @Nullable FileId filename, String message, @Nullable Throwable cause) {
+    public LexException(int line, int column, @Nullable FileId filename, String message, @Nullable Throwable cause) {
         super(message, cause);
         this.line = max(line, 1);
         this.column = max(column, 1);
@@ -44,7 +46,7 @@ public final class TokenMgrError extends FileAnalysisException {
      * Constructor called by JavaCC.
      */
     @InternalApi
-    public TokenMgrError(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
+    public LexException(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
         super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
         line = max(errorLine, 1);
         column = max(errorColumn, 1);
@@ -76,7 +78,7 @@ public final class TokenMgrError extends FileAnalysisException {
      * @return A new exception
      */
     @Override
-    public TokenMgrError setFileId(FileId fileId) {
+    public LexException setFileId(FileId fileId) {
         super.setFileId(fileId);
         return this;
     }

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java

@@ -10,7 +10,7 @@ import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 
 import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
 /**
@@ -63,7 +63,7 @@ public class AntlrTokenManager implements TokenManager<AntlrToken> {
                                 final int charPositionInLine,
                                 final String msg,
                                 final RecognitionException ex) {
-            throw new TokenMgrError(line, charPositionInLine, textDoc.getFileId(), msg, ex);
+            throw new LexException(line, charPositionInLine, textDoc.getFileId(), msg, ex);
         }
     }
 

pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/LexExceptionTest.java

@@ -8,30 +8,30 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
 
-class TokenMgrErrorTest {
+class LexExceptionTest {
     @Test
     void invalidLocation() {
-        TokenMgrError error = new TokenMgrError(2, 0, null, "test", null);
+        LexException error = new LexException(2, 0, null, "test", null);
         // this shouldn't throw a IllegalArgumentException
         assertEquals("line 2, column 1", error.location().startPosToString());
     }
 
     @Test
     void invalidLocationJavaCC() {
-        TokenMgrError error = new TokenMgrError(false, "DEFAULT", 2, 0, "}", '\n');
+        LexException error = new LexException(false, "DEFAULT", 2, 0, "}", '\n');
         // this shouldn't throw a IllegalArgumentException
         assertEquals("line 2, column 1", error.location().startPosToString());
     }
 
     @Test
     void validLocation() {
-        TokenMgrError error = new TokenMgrError(1, 1, null, "test", null);
+        LexException error = new LexException(1, 1, null, "test", null);
         assertEquals("line 1, column 1", error.location().startPosToString());
     }
 
     @Test
     void validLocationJavaCC() {
-        TokenMgrError error = new TokenMgrError(false, "DEFAULT", 1, 1, "}", '\n');
+        LexException error = new LexException(false, "DEFAULT", 1, 1, "}", '\n');
         assertEquals("line 1, column 1", error.location().startPosToString());
     }
 }

pmd-cpp/etc/grammar/Cpp.jj

@@ -32,9 +32,6 @@ options {
 PARSER_BEGIN(CppParserImpl)
 package net.sourceforge.pmd.lang.cpp.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 public final class CppParserImpl {
 
 }

pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/cpd/CPPTokenizerTest.java

@@ -71,7 +71,7 @@ class CPPTokenizerTest extends CpdTextComparisonTest {
 
     @Test
     void testWrongUnicodeInIdentifier() {
-        expectTokenMgrError(" void main() { int ⚜ = __; }");
+        expectLexException(" void main() { int ⚜ = __; }");
     }
 
     @Test
@@ -107,7 +107,7 @@ class CPPTokenizerTest extends CpdTextComparisonTest {
 
     @Test
     void testLexicalErrorFilename() {
-        expectTokenMgrError(sourceText("issue-1559"), dontSkipBlocks());
+        expectLexException(sourceText("issue-1559"), dontSkipBlocks());
     }
 
 

pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java

@@ -12,7 +12,7 @@ import org.junit.jupiter.api.Test;
 import net.sourceforge.pmd.cpd.CpdLanguageProperties;
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
 import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 
 class CsTokenizerTest extends CpdTextComparisonTest {
 
@@ -37,7 +37,7 @@ class CsTokenizerTest extends CpdTextComparisonTest {
 
     @Test
     void testOpenString() {
-        assertThrows(TokenMgrError.class, () -> doTest("unlexable_string"));
+        assertThrows(LexException.class, () -> doTest("unlexable_string"));
     }
 
     @Test

pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/ast/impl/antlr4/GroovyTokenManager.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.groovy.ast.impl.antlr4;
 import org.apache.groovy.parser.antlr4.GroovyLexer;
 
 import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
@@ -81,7 +81,7 @@ public class GroovyTokenManager implements TokenManager<GroovyToken> {
                                 final int charPositionInLine,
                                 final String msg,
                                 final RecognitionException ex) {
-            throw new TokenMgrError(line, charPositionInLine, textDoc.getFileId(), msg, ex);
+            throw new LexException(line, charPositionInLine, textDoc.getFileId(), msg, ex);
         }
     }
 

pmd-java/etc/grammar/Java.jjt

@@ -301,12 +301,8 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
 import java.util.Map;
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.GenericToken;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
 import net.sourceforge.pmd.lang.ast.Node;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-import net.sourceforge.pmd.lang.ast.Node;
 import net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.TokenContext;
 import net.sourceforge.pmd.lang.java.types.JPrimitiveType.PrimitiveTypeKind;
 

pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java

@@ -16,7 +16,7 @@ import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.Tokens;
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
 import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.document.FileId;
 import net.sourceforge.pmd.lang.document.TextDocument;
 import net.sourceforge.pmd.lang.java.JavaLanguageModule;
@@ -42,7 +42,7 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
     void testLexExceptionLocation() {
         Tokenizer tokenizer = newTokenizer(defaultProperties());
         Tokens tokens = new Tokens();
-        TokenMgrError lexException = assertThrows(TokenMgrError.class, () ->
+        LexException lexException = assertThrows(LexException.class, () ->
             Tokenizer.tokenize(tokenizer,
                     // note: the source deliberately contains an unbalanced quote, unterminated string literal
                     TextDocument.readOnlyString("class F {\n    String s=\"abc\";\"\n}\n", FileId.UNKNOWN, getLanguage().getDefaultVersion()),

pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt

@@ -270,7 +270,7 @@ open class ParserTestCtx(testScope: TestScope,
                 Pair(true, null)
             } catch (e: ParseException) {
                 Pair(false, e)
-            } catch (e: TokenMgrError) {
+            } catch (e: LexException) {
                 Pair(false, e)
             }
 

pmd-javascript/etc/grammar/Ecmascript5.jj

@@ -15,9 +15,6 @@ options {
 PARSER_BEGIN(Ecmascript5ParserImpl)
 package net.sourceforge.pmd.lang.ecmascript5.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 public class Ecmascript5ParserImpl {
 
 }

pmd-jsp/etc/grammar/Jsp.jjt

@@ -30,10 +30,6 @@ options {
 PARSER_BEGIN(JspParserImpl)
 package net.sourceforge.pmd.lang.jsp.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
-
 /**
  * JSP Parser for PMD.
  * @author Pieter, Application Engineers NV/SA, http://www.ae.be

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt

@@ -6,16 +6,13 @@ package net.sourceforge.pmd.cpd.test
 
 import io.kotest.assertions.throwables.shouldThrow
 import net.sourceforge.pmd.cpd.*
-import net.sourceforge.pmd.lang.Language
 import net.sourceforge.pmd.lang.LanguagePropertyBundle
 import net.sourceforge.pmd.lang.LanguageRegistry
-import net.sourceforge.pmd.lang.ast.TokenMgrError
+import net.sourceforge.pmd.lang.ast.LexException
 import net.sourceforge.pmd.lang.document.TextDocument
-import net.sourceforge.pmd.lang.document.TextFile
 import net.sourceforge.pmd.lang.document.FileId
 import net.sourceforge.pmd.test.BaseTextComparisonTest
 import org.apache.commons.lang3.StringUtils
-import java.util.*
 
 /**
  * CPD test comparing a dump of a file against a saved baseline.
@@ -73,18 +70,18 @@ abstract class CpdTextComparisonTest(
     }
 
     @JvmOverloads
-    fun expectTokenMgrError(
+    fun expectLexException(
         source: String,
         fileName: FileId = FileId.UNKNOWN,
         properties: LanguagePropertyConfig = defaultProperties()
-    ): TokenMgrError =
+    ): LexException =
-        expectTokenMgrError(FileData(fileName, source), properties)
+        expectLexException(FileData(fileName, source), properties)
 
     @JvmOverloads
-    fun expectTokenMgrError(
+    fun expectLexException(
         fileData: FileData,
         config: LanguagePropertyConfig = defaultProperties()
-    ): TokenMgrError =
+    ): LexException =
         shouldThrow {
             tokenize(newTokenizer(config), fileData)
         }

pmd-matlab/etc/grammar/Matlab.jj

@@ -21,9 +21,6 @@ options {
 PARSER_BEGIN(MatlabParserImpl)
 package net.sourceforge.pmd.lang.matlab.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 public class MatlabParserImpl {
 
 }

pmd-modelica/etc/grammar/Modelica.jjt

@@ -49,9 +49,6 @@ options {
 PARSER_BEGIN(ModelicaParserImpl)
 package net.sourceforge.pmd.lang.modelica.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 class ModelicaParserImpl {
 
 }

pmd-objectivec/etc/grammar/ObjectiveC.jj

@@ -18,12 +18,6 @@ options {
 PARSER_BEGIN(ObjectiveCParserImpl)
 package net.sourceforge.pmd.lang.objectivec.ast;
 
-import java.io.*;
-import java.util.*;
-
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 /**
  * Grammar to parse ObjectiveC 2.0
  * @author Michael Hall - Based on Parnel SableCC ObjectiveC grammar and javacc C grammar

pmd-plsql/etc/grammar/PLSQL.jjt

@@ -157,13 +157,11 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
 package net.sourceforge.pmd.lang.plsql.ast;
 
-import java.util.List;
-import java.util.ArrayList;
-import net.sourceforge.pmd.lang.ast.Node;
 import net.sourceforge.pmd.lang.document.Chars;
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.plsql.ast.internal.ParsingExclusion;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+
+import java.util.ArrayList;
+import java.util.List;
 
 public class PLSQLParserImpl {
 

pmd-python/etc/grammar/Python.jj

@@ -17,9 +17,6 @@ PARSER_BEGIN(PythonParserImpl)
 
 package net.sourceforge.pmd.lang.python.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 public class PythonParserImpl {
 
 }

pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java

@@ -9,7 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import org.junit.jupiter.api.Test;
 
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.LexException;
 import net.sourceforge.pmd.lang.scala.ScalaLanguageModule;
 
 class ScalaTokenizerTest extends CpdTextComparisonTest {
@@ -35,7 +35,7 @@ class ScalaTokenizerTest extends CpdTextComparisonTest {
 
     @Test
     void tokenizeFailTest() {
-        assertThrows(TokenMgrError.class, () -> doTest("unlexable_sample"));
+        assertThrows(LexException.class, () -> doTest("unlexable_sample"));
     }
 
     @Test

pmd-visualforce/etc/grammar/Vf.jjt

@@ -13,9 +13,6 @@ options {
 PARSER_BEGIN(VfParserImpl)
 package net.sourceforge.pmd.lang.vf.ast;
 
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-
 public class VfParserImpl {
 
 

pmd-vm/etc/grammar/Vm.jjt

@@ -37,16 +37,11 @@ PARSER_BEGIN(VmParserImpl)
 package net.sourceforge.pmd.lang.vm.ast;
 
 import java.io.IOException;
-import java.io.Reader;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.HashMap;
-import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;