diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java index 5e55743087..7d8462e164 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java @@ -11,9 +11,8 @@ import net.sourceforge.pmd.cpd.SourceCode; import net.sourceforge.pmd.cpd.TokenEntry; import net.sourceforge.pmd.cpd.Tokenizer; import net.sourceforge.pmd.cpd.Tokens; -import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; -import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; /** @@ -32,13 +31,9 @@ public abstract class AntlrTokenizer implements Tokenizer { try { AntlrToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { - processToken(tokenEntries, tokenManager.getFileName(), currentToken); + processToken(tokenEntries, sourceCode.getFileName(), currentToken); currentToken = tokenFilter.getNextToken(); } - } catch (final AntlrTokenManager.ANTLRSyntaxError err) { - // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled - // when CPD is executed with the '--skipLexicalErrors' command line option - throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), null); } finally { tokenEntries.add(TokenEntry.getEOF()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java index f358bddfc9..259e3df2fa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd.token; import static org.antlr.v4.runtime.Token.EOF; import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; /** @@ -25,6 +26,6 @@ public class AntlrTokenFilter extends BaseTokenFilter { @Override protected boolean shouldStopProcessing(final AntlrToken currentToken) { - return currentToken.getType() == EOF; + return currentToken.getKind() == EOF; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java index f68022199b..477b8de445 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java @@ -18,6 +18,15 @@ public final class TokenMgrError extends RuntimeException { private final int column; private final String filename; + /** + * Create a new exception. + * + * @param line Line number + * @param column Column number + * @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later + * @param message Message of the error + * @param cause Cause of the error, if any + */ public TokenMgrError(int line, int column, @Nullable String filename, String message, @Nullable Throwable cause) { super(message, cause); this.line = line; @@ -29,7 +38,6 @@ public final class TokenMgrError extends RuntimeException { * Constructor called by JavaCC. */ @InternalApi - @SuppressWarnings("PMD.UnusedFormalParameter") public TokenMgrError(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) { super(makeReason(eofSeen, lexStateName, errorAfter, curChar)); line = errorLine; @@ -60,6 +68,8 @@ public final class TokenMgrError extends RuntimeException { * Replace the file name of this error. * * @param filename New filename + * + * @return A new exception */ public TokenMgrError withFileName(String filename) { return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause()); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java similarity index 95% rename from pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java index e4b4d23426..0ab3ca442a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java @@ -1,8 +1,8 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd.token; +package net.sourceforge.pmd.lang.ast.impl.antlr4; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Token; @@ -64,7 +64,7 @@ public class AntlrToken implements GenericToken { return token.getCharPositionInLine() + token.getStopIndex() - token.getStartIndex(); } - public int getType() { + public int getKind() { return token.getType(); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java index aaf7150a2b..6333f78a72 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java @@ -9,15 +9,16 @@ import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; -import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.TokenMgrError; /** * Generic token manager implementation for all Antlr lexers. */ public class AntlrTokenManager implements TokenManager { + private final Lexer lexer; - private String fileName; + private final String fileName; private AntlrToken previousToken; /** @@ -58,33 +59,17 @@ public class AntlrTokenManager implements TokenManager { lexer.addErrorListener(new ErrorHandler()); } - private static class ErrorHandler extends BaseErrorListener { + private class ErrorHandler extends BaseErrorListener { @Override - public void syntaxError(final Recognizer recognizer, final Object offendingSymbol, final int line, - final int charPositionInLine, final String msg, final RecognitionException ex) { - throw new ANTLRSyntaxError(msg, line, charPositionInLine, ex); + public void syntaxError(final Recognizer recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String msg, + final RecognitionException ex) { + throw new TokenMgrError(line, charPositionInLine, getFileName(), msg, ex); } } - public static class ANTLRSyntaxError extends RuntimeException { - private static final long serialVersionUID = 1L; - private final int line; - private final int column; - - /* default */ ANTLRSyntaxError(final String msg, final int line, final int column, - final RecognitionException cause) { - super(msg, cause); - this.line = line; - this.column = column; - } - - public int getLine() { - return line; - } - - public int getColumn() { - return column; - } - } } diff --git a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java index c1f797f1c6..1209a82ac6 100644 --- a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java +++ b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java @@ -9,7 +9,7 @@ import java.util.Properties; import org.antlr.v4.runtime.CharStream; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; -import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.lang.cs.antlr4.CSharpLexer; @@ -79,7 +79,7 @@ public class CsTokenizer extends AntlrTokenizer { private void skipUsingDirectives(final AntlrToken currentToken, final Iterable remainingTokens) { if (ignoreUsings) { - final int type = currentToken.getType(); + final int type = currentToken.getKind(); if (type == CSharpLexer.USING && isUsingDirective(remainingTokens)) { discardingUsings = true; } else if (type == CSharpLexer.SEMICOLON && discardingUsings) { @@ -92,7 +92,7 @@ public class CsTokenizer extends AntlrTokenizer { private boolean isUsingDirective(final Iterable remainingTokens) { UsingState usingState = UsingState.KEYWORD; for (final AntlrToken token : remainingTokens) { - final int type = token.getType(); + final int type = token.getKind(); if (usingState == UsingState.KEYWORD) { // The previous token was a using keyword. switch (type) { @@ -148,7 +148,7 @@ public class CsTokenizer extends AntlrTokenizer { } private void skipNewLines(final AntlrToken currentToken) { - discardingNL = currentToken.getType() == CSharpLexer.NL; + discardingNL = currentToken.getKind() == CSharpLexer.NL; } @Override diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java index 9afb02b1dd..18a258093f 100644 --- a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; -import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.lang.dart.antlr4.Dart2Lexer; @@ -53,7 +53,7 @@ public class DartTokenizer extends AntlrTokenizer { } private void skipLibraryAndImport(final AntlrToken currentToken) { - final int type = currentToken.getType(); + final int type = currentToken.getKind(); if (type == Dart2Lexer.LIBRARY || type == Dart2Lexer.IMPORT) { discardingLibraryAndImport = true; } else if (discardingLibraryAndImport && (type == Dart2Lexer.SEMICOLON || type == Dart2Lexer.NEWLINE)) { @@ -62,11 +62,11 @@ public class DartTokenizer extends AntlrTokenizer { } private void skipNewLines(final AntlrToken currentToken) { - discardingNL = currentToken.getType() == Dart2Lexer.NEWLINE; + discardingNL = currentToken.getKind() == Dart2Lexer.NEWLINE; } private void skipSemicolons(final AntlrToken currentToken) { - discardingSemicolon = currentToken.getType() == Dart2Lexer.SEMICOLON; + discardingSemicolon = currentToken.getKind() == Dart2Lexer.SEMICOLON; } @Override diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index 963ea09c3c..987ab6268f 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; import net.sourceforge.pmd.cpd.internal.AntlrTokenizer; -import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; @@ -51,7 +51,7 @@ public class KotlinTokenizer extends AntlrTokenizer { } private void skipPackageAndImport(final AntlrToken currentToken) { - final int type = currentToken.getType(); + final int type = currentToken.getKind(); if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { discardingPackageAndImport = true; } else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { @@ -60,7 +60,7 @@ public class KotlinTokenizer extends AntlrTokenizer { } private void skipNewLines(final AntlrToken currentToken) { - discardingNL = currentToken.getType() == Kotlin.NL; + discardingNL = currentToken.getKind() == Kotlin.NL; } @Override