phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Add a generic JavaCCTokenizer for CPD

Browse Source
This commit is contained in:
Andreas Dangel
2019-01-26 11:24:32 +01:00
parent a79d4aeb4f
commit 08b7dd49dd
13 changed files with 154 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@@ -4,38 +4,20 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.python.PythonTokenManager;
import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Python tokenizer.
*/
public class PythonTokenizer implements Tokenizer {
public class PythonTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
PythonTokenManager tokenManager = new PythonTokenManager(reader);
tokenManager.setFileName(sourceCode.getFileName());
TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
} catch (IOException err) {
err.printStackTrace();
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
return new PythonTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}