hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

python

Browse Source
This commit is contained in:
Clément Fournier
2020-03-20 16:41:54 +01:00
parent cd5af6a12d
commit f64acb6c38
2 changed files with 26 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -4,15 +4,18 @@
package net.sourceforge.pmd.cpd;
import java.io.StringReader;
import java.io.Reader;
import java.util.regex.Pattern;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds;
import net.sourceforge.pmd.lang.python.ast.PythonTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Python tokenizer.
@ -22,9 +25,26 @@ public class PythonTokenizer extends JavaCCTokenizer {
private static final Pattern STRING_NL_ESCAPE = Pattern.compile("\\\\\\r?\\n");
@Override
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
return new PythonTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
protected TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode) {
return PythonTokenKinds.newTokenManager(sourceCode);
}
@Override
protected CharStream makeCharStream(Reader sourceCode) {
return CharStreamFactory.simpleCharStream(sourceCode, PythonTokenDocument::new);
}
private static class PythonTokenDocument extends JavaccTokenDocument {
PythonTokenDocument(String fullText) {
super(fullText);
}
@Override
protected @Nullable String describeKindImpl(int kind) {
return PythonTokenKinds.describe(kind);
}
}
@Override

48
pmd-python/src/main/java/net/sourceforge/pmd/lang/python/ast/PythonTokenManager.java
View File

@ -1,48 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.python.ast;
import java.io.Reader;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
/**
* Python Token Manager implementation.
*/
public class PythonTokenManager implements TokenManager {
private final PythonParserImplTokenManager tokenManager;
/**
* Creates a new Python Token Manager from the given source code.
*
* @param source
* the source code
*/
public PythonTokenManager(Reader source) {
tokenManager = new PythonParserImplTokenManager(CharStreamFactory.simpleCharStream(source, PythonTokenDocument::new));
}
@Override
public Object getNextToken() {
return tokenManager.getNextToken();
}
private static class PythonTokenDocument extends JavaccTokenDocument {
PythonTokenDocument(String fullText) {
super(fullText);
}
@Override
protected @Nullable String describeKindImpl(int kind) {
return PythonTokenKinds.describe(kind);
}
}
}