hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Refactor token documents

Browse Source 
Isolate static parts
This commit is contained in:
Clément Fournier
2020-09-02 08:41:23 +02:00
parent a01f1a3574
commit 3ee32effda
25 changed files with 296 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -13,6 +13,7 @@ import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior;
import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds;
import net.sourceforge.pmd.util.document.TextDocument;
@ -23,27 +24,16 @@ public class PythonTokenizer extends JavaCCTokenizer {
private static final Pattern STRING_NL_ESCAPE = Pattern.compile("\\\\\\r?\\n");
private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(PythonTokenKinds.TOKEN_NAMES);
@Override
protected TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode) {
return PythonTokenKinds.newTokenManager(sourceCode);
}
@Override
protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) {
return new PythonTokenDocument(textDoc);
}
private static class PythonTokenDocument extends JavaccTokenDocument {
PythonTokenDocument(TextDocument fullText) {
super(fullText);
}
@Override
protected @Nullable String describeKindImpl(int kind) {
return PythonTokenKinds.describe(kind);
}
protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() {
return TOKEN_BEHAVIOR;
}
@Override