hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

CPD: Fix error handling for lexical errors

Browse Source 
* TokenMgrError must not be caught by the tokenizer. This is handled
  by CPD itself
* The token managers need to know the filename for proper error messages
This commit is contained in:
Andreas Dangel
2019-01-20 09:59:59 +01:00
parent a8e1364072
commit b745f331b8
5 changed files with 33 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -10,10 +10,7 @@ import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import net.sourceforge.pmd.lang.python.PythonTokenManager;
import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
@ -26,21 +23,18 @@ public class PythonTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader));
PythonTokenManager tokenManager = new PythonTokenManager(reader);
tokenManager.setFileName(sourceCode.getFileName());
TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode);
} catch (TokenMgrError | IOException err) {
} catch (IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode + " due to parse error");
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
}