Make Antlr token manager throw TokenMgrError directly

pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java

@@ -11,9 +11,8 @@ import net.sourceforge.pmd.cpd.SourceCode;
 import net.sourceforge.pmd.cpd.TokenEntry;
 import net.sourceforge.pmd.cpd.Tokenizer;
 import net.sourceforge.pmd.cpd.Tokens;
-import net.sourceforge.pmd.cpd.token.AntlrToken;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 
 /**
@@ -32,13 +31,9 @@ public abstract class AntlrTokenizer implements Tokenizer {
         try {
             AntlrToken currentToken = tokenFilter.getNextToken();
             while (currentToken != null) {
-                processToken(tokenEntries, tokenManager.getFileName(), currentToken);
+                processToken(tokenEntries, sourceCode.getFileName(), currentToken);
                 currentToken = tokenFilter.getNextToken();
             }
-        } catch (final AntlrTokenManager.ANTLRSyntaxError err) {
-            // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
-            // when CPD is executed with the '--skipLexicalErrors' command line option
-            throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), null);
         } finally {
             tokenEntries.add(TokenEntry.getEOF());
         }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java

@@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd.token;
 import static org.antlr.v4.runtime.Token.EOF;
 
 import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
+import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 
 /**
@@ -25,6 +26,6 @@ public class AntlrTokenFilter extends BaseTokenFilter<AntlrToken> {
 
     @Override
     protected boolean shouldStopProcessing(final AntlrToken currentToken) {
-        return currentToken.getType() == EOF;
+        return currentToken.getKind() == EOF;
     }
 }

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java

@@ -18,6 +18,15 @@ public final class TokenMgrError extends RuntimeException {
     private final int column;
     private final String filename;
 
+    /**
+     * Create a new exception.
+     *
+     * @param line     Line number
+     * @param column   Column number
+     * @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later
+     * @param message  Message of the error
+     * @param cause    Cause of the error, if any
+     */
     public TokenMgrError(int line, int column, @Nullable String filename, String message, @Nullable Throwable cause) {
         super(message, cause);
         this.line = line;
@@ -29,7 +38,6 @@ public final class TokenMgrError extends RuntimeException {
      * Constructor called by JavaCC.
      */
     @InternalApi
-    @SuppressWarnings("PMD.UnusedFormalParameter")
     public TokenMgrError(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
         super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
         line = errorLine;
@@ -60,6 +68,8 @@ public final class TokenMgrError extends RuntimeException {
      * Replace the file name of this error.
      *
      * @param filename New filename
+     *
+     * @return A new exception
      */
     public TokenMgrError withFileName(String filename) {
         return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause());

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java

@@ -1,8 +1,8 @@
-/**
+/*
  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  */
 
-package net.sourceforge.pmd.cpd.token;
+package net.sourceforge.pmd.lang.ast.impl.antlr4;
 
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.Token;
@@ -64,7 +64,7 @@ public class AntlrToken implements GenericToken<AntlrToken> {
         return token.getCharPositionInLine() + token.getStopIndex() - token.getStartIndex();
     }
 
-    public int getType() {
+    public int getKind() {
         return token.getType();
     }
 

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java

@@ -9,15 +9,16 @@ import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 
-import net.sourceforge.pmd.cpd.token.AntlrToken;
 import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 /**
  * Generic token manager implementation for all Antlr lexers.
  */
 public class AntlrTokenManager implements TokenManager<AntlrToken> {
+
     private final Lexer lexer;
-    private String fileName;
+    private final String fileName;
     private AntlrToken previousToken;
 
     /**
@@ -58,33 +59,17 @@ public class AntlrTokenManager implements TokenManager<AntlrToken> {
         lexer.addErrorListener(new ErrorHandler());
     }
 
-    private static class ErrorHandler extends BaseErrorListener {
+    private class ErrorHandler extends BaseErrorListener {
 
         @Override
-        public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
+        public void syntaxError(final Recognizer<?, ?> recognizer,
-            final int charPositionInLine, final String msg, final RecognitionException ex) {
+                                final Object offendingSymbol,
-            throw new ANTLRSyntaxError(msg, line, charPositionInLine, ex);
+                                final int line,
+                                final int charPositionInLine,
+                                final String msg,
+                                final RecognitionException ex) {
+            throw new TokenMgrError(line, charPositionInLine, getFileName(), msg, ex);
         }
     }
 
-    public static class ANTLRSyntaxError extends RuntimeException {
-        private static final long serialVersionUID = 1L;
-        private final int line;
-        private final int column;
-
-        /* default */ ANTLRSyntaxError(final String msg, final int line, final int column,
-            final RecognitionException cause) {
-            super(msg, cause);
-            this.line = line;
-            this.column = column;
-        }
-
-        public int getLine() {
-            return line;
-        }
-
-        public int getColumn() {
-            return column;
-        }
-    }
 }

pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java

@@ -9,7 +9,7 @@ import java.util.Properties;
 import org.antlr.v4.runtime.CharStream;
 
 import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
-import net.sourceforge.pmd.cpd.token.AntlrToken;
+import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 import net.sourceforge.pmd.lang.cs.antlr4.CSharpLexer;
@@ -79,7 +79,7 @@ public class CsTokenizer extends AntlrTokenizer {
 
         private void skipUsingDirectives(final AntlrToken currentToken, final Iterable<AntlrToken> remainingTokens) {
             if (ignoreUsings) {
-                final int type = currentToken.getType();
+                final int type = currentToken.getKind();
                 if (type == CSharpLexer.USING && isUsingDirective(remainingTokens)) {
                     discardingUsings = true;
                 } else if (type == CSharpLexer.SEMICOLON && discardingUsings) {
@@ -92,7 +92,7 @@ public class CsTokenizer extends AntlrTokenizer {
         private boolean isUsingDirective(final Iterable<AntlrToken> remainingTokens) {
             UsingState usingState = UsingState.KEYWORD;
             for (final AntlrToken token : remainingTokens) {
-                final int type = token.getType();
+                final int type = token.getKind();
                 if (usingState == UsingState.KEYWORD) {
                     // The previous token was a using keyword.
                     switch (type) {
@@ -148,7 +148,7 @@ public class CsTokenizer extends AntlrTokenizer {
         }
 
         private void skipNewLines(final AntlrToken currentToken) {
-            discardingNL = currentToken.getType() == CSharpLexer.NL;
+            discardingNL = currentToken.getKind() == CSharpLexer.NL;
         }
 
         @Override

pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 
 import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
-import net.sourceforge.pmd.cpd.token.AntlrToken;
+import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 import net.sourceforge.pmd.lang.dart.antlr4.Dart2Lexer;
@@ -53,7 +53,7 @@ public class DartTokenizer extends AntlrTokenizer {
         }
 
         private void skipLibraryAndImport(final AntlrToken currentToken) {
-            final int type = currentToken.getType();
+            final int type = currentToken.getKind();
             if (type == Dart2Lexer.LIBRARY || type == Dart2Lexer.IMPORT) {
                 discardingLibraryAndImport = true;
             } else if (discardingLibraryAndImport && (type == Dart2Lexer.SEMICOLON || type == Dart2Lexer.NEWLINE)) {
@@ -62,11 +62,11 @@ public class DartTokenizer extends AntlrTokenizer {
         }
 
         private void skipNewLines(final AntlrToken currentToken) {
-            discardingNL = currentToken.getType() == Dart2Lexer.NEWLINE;
+            discardingNL = currentToken.getKind() == Dart2Lexer.NEWLINE;
         }
 
         private void skipSemicolons(final AntlrToken currentToken) {
-            discardingSemicolon = currentToken.getType() == Dart2Lexer.SEMICOLON;
+            discardingSemicolon = currentToken.getKind() == Dart2Lexer.SEMICOLON;
         }
 
         @Override

pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java

@@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
 import org.antlr.v4.runtime.CharStream;
 
 import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
-import net.sourceforge.pmd.cpd.token.AntlrToken;
+import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
 import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
 import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
 import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin;
@@ -51,7 +51,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
         }
 
         private void skipPackageAndImport(final AntlrToken currentToken) {
-            final int type = currentToken.getType();
+            final int type = currentToken.getKind();
             if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) {
                 discardingPackageAndImport = true;
             } else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) {
@@ -60,7 +60,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
         }
 
         private void skipNewLines(final AntlrToken currentToken) {
-            discardingNL = currentToken.getType() == Kotlin.NL;
+            discardingNL = currentToken.getKind() == Kotlin.NL;
         }
 
         @Override