phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into 7.0.x

Browse Source
This commit is contained in:
Clément Fournier
2019-02-18 19:32:55 +01:00
parent 3bfa33ab0d c6375185c2
commit 6d84530d52
103 changed files with 2883 additions and 776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -4,44 +4,21 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.python.PythonTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Python tokenizer.
*/
public class PythonTokenizer implements Tokenizer {
public class PythonTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode);
} catch (TokenMgrError | IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
}
return new PythonTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}