hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Make Antlr token manager throw TokenMgrError directly

Browse Source
This commit is contained in:
Clément Fournier 2020-03-22 00:45:46 +01:00
parent 62a9d5b655
commit 16f30c9ece
8 changed files with 40 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java
View File

@ -11,9 +11,8 @@ import net.sourceforge.pmd.cpd.SourceCode;
import net.sourceforge.pmd.cpd.TokenEntry;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
/**
@ -32,13 +31,9 @@ public abstract class AntlrTokenizer implements Tokenizer {
try {
AntlrToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
processToken(tokenEntries, tokenManager.getFileName(), currentToken);
processToken(tokenEntries, sourceCode.getFileName(), currentToken);
currentToken = tokenFilter.getNextToken();
}
} catch (final AntlrTokenManager.ANTLRSyntaxError err) {
// Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
// when CPD is executed with the '--skipLexicalErrors' command line option
throw new TokenMgrError(err.getLine(), err.getColumn(), tokenManager.getFileName(), err.getMessage(), null);
} finally {
tokenEntries.add(TokenEntry.getEOF());
}

3
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd.token;
import static org.antlr.v4.runtime.Token.EOF;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
/**
@ -25,6 +26,6 @@ public class AntlrTokenFilter extends BaseTokenFilter<AntlrToken> {
@Override
protected boolean shouldStopProcessing(final AntlrToken currentToken) {
return currentToken.getType() == EOF;
return currentToken.getKind() == EOF;
}
}

12
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java
View File

@ -18,6 +18,15 @@ public final class TokenMgrError extends RuntimeException {
private final int column;
private final String filename;
/**
* Create a new exception.
*
* @param line Line number
* @param column Column number
* @param filename Filename. If unknown, it can be completed with {@link #withFileName(String)} later
* @param message Message of the error
* @param cause Cause of the error, if any
*/
public TokenMgrError(int line, int column, @Nullable String filename, String message, @Nullable Throwable cause) {
super(message, cause);
this.line = line;
@ -29,7 +38,6 @@ public final class TokenMgrError extends RuntimeException {
* Constructor called by JavaCC.
*/
@InternalApi
@SuppressWarnings("PMD.UnusedFormalParameter")
public TokenMgrError(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
line = errorLine;
@ -60,6 +68,8 @@ public final class TokenMgrError extends RuntimeException {
* Replace the file name of this error.
*
* @param filename New filename
*
* @return A new exception
*/
public TokenMgrError withFileName(String filename) {
return new TokenMgrError(this.line, this.column, filename, this.getMessage(), this.getCause());

6
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java → pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrToken.java
View File

@ -1,8 +1,8 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token;
package net.sourceforge.pmd.lang.ast.impl.antlr4;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
@ -64,7 +64,7 @@ public class AntlrToken implements GenericToken<AntlrToken> {
return token.getCharPositionInLine() + token.getStopIndex() - token.getStartIndex();
}
public int getType() {
public int getKind() {
return token.getType();
}

37
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java
View File

@ -9,15 +9,16 @@ import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Generic token manager implementation for all Antlr lexers.
*/
public class AntlrTokenManager implements TokenManager<AntlrToken> {
private final Lexer lexer;
private String fileName;
private final String fileName;
private AntlrToken previousToken;
/**
@ -58,33 +59,17 @@ public class AntlrTokenManager implements TokenManager<AntlrToken> {
lexer.addErrorListener(new ErrorHandler());
}
private static class ErrorHandler extends BaseErrorListener {
private class ErrorHandler extends BaseErrorListener {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException ex) {
throw new ANTLRSyntaxError(msg, line, charPositionInLine, ex);
public void syntaxError(final Recognizer<?, ?> recognizer,
final Object offendingSymbol,
final int line,
final int charPositionInLine,
final String msg,
final RecognitionException ex) {
throw new TokenMgrError(line, charPositionInLine, getFileName(), msg, ex);
}
}
public static class ANTLRSyntaxError extends RuntimeException {
private static final long serialVersionUID = 1L;
private final int line;
private final int column;
/* default */ ANTLRSyntaxError(final String msg, final int line, final int column,
final RecognitionException cause) {
super(msg, cause);
this.line = line;
this.column = column;
}
public int getLine() {
return line;
}
public int getColumn() {
return column;
}
}
}

8
pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java
View File

@ -9,7 +9,7 @@ import java.util.Properties;
import org.antlr.v4.runtime.CharStream;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.cs.antlr4.CSharpLexer;
@ -79,7 +79,7 @@ public class CsTokenizer extends AntlrTokenizer {
private void skipUsingDirectives(final AntlrToken currentToken, final Iterable<AntlrToken> remainingTokens) {
if (ignoreUsings) {
final int type = currentToken.getType();
final int type = currentToken.getKind();
if (type == CSharpLexer.USING && isUsingDirective(remainingTokens)) {
discardingUsings = true;
} else if (type == CSharpLexer.SEMICOLON && discardingUsings) {
@ -92,7 +92,7 @@ public class CsTokenizer extends AntlrTokenizer {
private boolean isUsingDirective(final Iterable<AntlrToken> remainingTokens) {
UsingState usingState = UsingState.KEYWORD;
for (final AntlrToken token : remainingTokens) {
final int type = token.getType();
final int type = token.getKind();
if (usingState == UsingState.KEYWORD) {
// The previous token was a using keyword.
switch (type) {
@ -148,7 +148,7 @@ public class CsTokenizer extends AntlrTokenizer {
}
private void skipNewLines(final AntlrToken currentToken) {
discardingNL = currentToken.getType() == CSharpLexer.NL;
discardingNL = currentToken.getKind() == CSharpLexer.NL;
}
@Override

8
pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.dart.antlr4.Dart2Lexer;
@ -53,7 +53,7 @@ public class DartTokenizer extends AntlrTokenizer {
}
private void skipLibraryAndImport(final AntlrToken currentToken) {
final int type = currentToken.getType();
final int type = currentToken.getKind();
if (type == Dart2Lexer.LIBRARY || type == Dart2Lexer.IMPORT) {
discardingLibraryAndImport = true;
} else if (discardingLibraryAndImport && (type == Dart2Lexer.SEMICOLON || type == Dart2Lexer.NEWLINE)) {
@ -62,11 +62,11 @@ public class DartTokenizer extends AntlrTokenizer {
}
private void skipNewLines(final AntlrToken currentToken) {
discardingNL = currentToken.getType() == Dart2Lexer.NEWLINE;
discardingNL = currentToken.getKind() == Dart2Lexer.NEWLINE;
}
private void skipSemicolons(final AntlrToken currentToken) {
discardingSemicolon = currentToken.getType() == Dart2Lexer.SEMICOLON;
discardingSemicolon = currentToken.getKind() == Dart2Lexer.SEMICOLON;
}
@Override

6
pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin;
@ -51,7 +51,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
}
private void skipPackageAndImport(final AntlrToken currentToken) {
final int type = currentToken.getType();
final int type = currentToken.getKind();
if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) {
discardingPackageAndImport = true;
} else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) {
@ -60,7 +60,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
}
private void skipNewLines(final AntlrToken currentToken) {
discardingNL = currentToken.getType() == Kotlin.NL;
discardingNL = currentToken.getKind() == Kotlin.NL;
}
@Override