hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

PR Review

Browse Source
This commit is contained in:
Tomi De Lucca
2018-10-14 19:02:15 -03:00
parent 9a91b4c29e
commit 7d65081841
4 changed files with 18 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
View File

@ -6,10 +6,9 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
import net.sourceforge.pmd.cpd.token.GenericAntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
@ -26,16 +25,16 @@ public abstract class AntlrTokenizer implements Tokenizer {
AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
try {
GenericAntlrToken token = (GenericAntlrToken) tokenManager.getNextToken();
AntlrToken token = (AntlrToken) tokenManager.getNextToken();
while (token.getType() != Token.EOF) {
if (token.getChannel() != Lexer.HIDDEN) {
if (!token.isHidden()) {
final TokenEntry tokenEntry =
new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine());
tokenEntries.add(tokenEntry);
}
token = (GenericAntlrToken) tokenManager.getNextToken();
token = (AntlrToken) tokenManager.getNextToken();
}
} catch (final AntlrTokenManager.ANTLRSyntaxError err) {
// Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled

11
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/GenericAntlrToken.java → pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java
View File

@ -4,6 +4,7 @@
package net.sourceforge.pmd.cpd.token;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
import net.sourceforge.pmd.lang.ast.GenericToken;
@ -13,10 +14,10 @@ import com.beust.jcommander.internal.Nullable;
/**
* Generic Antlr representation of a token.
*/
public class GenericAntlrToken implements GenericToken {
public class AntlrToken implements GenericToken {
private final Token token;
private final GenericAntlrToken previousComment;
private final AntlrToken previousComment;
/**
* Constructor
@ -24,7 +25,7 @@ public class GenericAntlrToken implements GenericToken {
* @param token The antlr token implementation
* @param previousComment The previous comment
*/
public GenericAntlrToken(final Token token, @Nullable final GenericAntlrToken previousComment) {
public AntlrToken(final Token token, @Nullable final AntlrToken previousComment) {
this.token = token;
this.previousComment = previousComment;
}
@ -69,7 +70,7 @@ public class GenericAntlrToken implements GenericToken {
return token.getType();
}
public int getChannel() {
return token.getChannel();
public boolean isHidden() {
return token.getChannel() == Lexer.HIDDEN;
}
}

26
pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
View File

@ -8,12 +8,8 @@ import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.Token;
import net.sourceforge.pmd.cpd.token.GenericAntlrToken;
import com.beust.jcommander.internal.Nullable;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.lang.TokenManager;
/**
@ -22,31 +18,27 @@ import net.sourceforge.pmd.lang.TokenManager;
public class AntlrTokenManager implements TokenManager {
private final Lexer lexer;
private String fileName;
private final String commentToken;
private GenericAntlrToken previousComment;
private AntlrToken previousToken;
/**
* Constructor
*
* @param lexer The lexer
* @param fileName The file name
* @param commentToken The list of all comment tokens on the grammar
*/
public AntlrTokenManager(final Lexer lexer, final String fileName, @Nullable final String commentToken) {
public AntlrTokenManager(final Lexer lexer, final String fileName) {
this.lexer = lexer;
this.fileName = fileName;
this.commentToken = commentToken;
resetListeners();
}
@Override
public Object getNextToken() {
final Token token = lexer.nextToken();
if (isCommentToken(token.getText())) {
previousComment = new GenericAntlrToken(token, previousComment);
}
final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null;
final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment);
previousToken = currentToken;
return new GenericAntlrToken(token, previousComment);
return currentToken;
}
@Override
@ -63,10 +55,6 @@ public class AntlrTokenManager implements TokenManager {
lexer.addErrorListener(new ErrorHandler());
}
private boolean isCommentToken(final String text) {
return commentToken != null && text != null && text.startsWith(commentToken);
}
private static class ErrorHandler extends BaseErrorListener {
@Override

4
pmd-swift/src/main/java/net/sourceforge/pmd/cpd/SwiftTokenizer.java
View File

@ -14,11 +14,9 @@ import net.sourceforge.pmd.lang.swift.antlr4.SwiftLexer;
*/
public class SwiftTokenizer extends AntlrTokenizer {
private static final String COMMENT_TOKEN = "//";
@Override
protected AntlrTokenManager getLexerForSource(final SourceCode sourceCode) {
CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode);
return new AntlrTokenManager(new SwiftLexer(charStream), sourceCode.getFileName(), COMMENT_TOKEN);
return new AntlrTokenManager(new SwiftLexer(charStream), sourceCode.getFileName());
}
}