hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Simplified Kotlin tokenizer.

Browse Source
This commit is contained in:
Maikel Steneker 2018-11-12 14:35:59 +01:00
parent 1dd9cfd17b
commit 166e17f483
1 changed files with 25 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java
View File

@ -4,101 +4,59 @@
package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.Token;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin;
/**
* The Kotlin Tokenizer
*/
public class KotlinTokenizer implements Tokenizer {
public class KotlinTokenizer extends AntlrTokenizer {
private boolean discardingPackageAndImport = false;
@Override
public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) {
final StringBuilder buffer = sourceCode.getCodeBuffer();
try {
final ANTLRInputStream ais = new ANTLRInputStream(buffer.toString());
final Kotlin lexer = new Kotlin(ais);
lexer.removeErrorListeners();
lexer.addErrorListener(new ErrorHandler());
Token token = lexer.nextToken();
while (token.getType() != Token.EOF) {
analyzeTokenStart(token);
if (token.getChannel() != Lexer.HIDDEN && token.getType() != Kotlin.NL && !isDiscarding()) {
final TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine());
tokenEntries.add(tokenEntry);
}
analyzeTokenEnd(token);
token = lexer.nextToken();
protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) {
CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode);
final Lexer lexer = new Kotlin(charStream);
final AntlrTokenManager tokenManager = new AntlrTokenManager(lexer, sourceCode.getFileName()) {
@Override
public Object getNextToken() {
AntlrToken nextToken;
boolean done = false;
do {
nextToken = (AntlrToken) super.getNextToken();
analyzeTokenStart(nextToken);
if (!nextToken.isHidden() && nextToken.getType() != Kotlin.NL && !isDiscarding()) {
done = true;
}
analyzeTokenEnd(nextToken);
} while (!done && nextToken.getType() != Token.EOF);
return nextToken;
}
} catch (final ANTLRSyntaxError err) {
// Wrap exceptions of the Kotlin tokenizer in a TokenMgrError, so
// they are correctly handled
// when CPD is executed with the '--skipLexicalErrors' command line
// option
throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + err.getLine()
+ ", column " + err.getColumn() + ". Encountered: " + err.getMessage(),
TokenMgrError.LEXICAL_ERROR);
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
};
return tokenManager;
}
private boolean isDiscarding() {
return discardingPackageAndImport;
}
private void analyzeTokenStart(final Token currentToken) {
private void analyzeTokenStart(final AntlrToken currentToken) {
final int type = currentToken.getType();
if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) {
discardingPackageAndImport = true;
}
}
private void analyzeTokenEnd(final Token currentToken) {
private void analyzeTokenEnd(final AntlrToken currentToken) {
final int type = currentToken.getType();
if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) {
discardingPackageAndImport = false;
}
}
private static class ErrorHandler extends BaseErrorListener {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine,
final String msg, final RecognitionException ex) {
throw new ANTLRSyntaxError(msg, line, charPositionInLine, ex);
}
}
private static class ANTLRSyntaxError extends RuntimeException {
private static final long serialVersionUID = 1L;
private final int line;
private final int column;
ANTLRSyntaxError(final String msg, final int line, final int column, final RecognitionException cause) {
super(msg, cause);
this.line = line;
this.column = column;
}
public int getLine() {
return line;
}
public int getColumn() {
return column;
}
}
}