phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CPD: Fix error handling for lexical errors

Browse Source 
* TokenMgrError must not be caught by the tokenizer. This is handled
  by CPD itself
* The token managers need to know the filename for proper error messages
This commit is contained in:
Andreas Dangel
2019-01-20 09:59:59 +01:00
parent a8e1364072
commit b745f331b8
5 changed files with 33 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
View File

@ -12,7 +12,6 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler; import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule; import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule;
import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants;
import net.sourceforge.pmd.lang.ecmascript5.ast.Token; import net.sourceforge.pmd.lang.ecmascript5.ast.Token;
@ -37,14 +36,10 @@ public class EcmascriptTokenizer implements Tokenizer {
new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine)); new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken(); currentToken = (Token) tokenFilter.getNextToken();
} }
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally {
tokenEntries.add(TokenEntry.getEOF());
} }
} }

11
pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
View File

@ -10,7 +10,6 @@ import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.matlab.MatlabTokenManager; import net.sourceforge.pmd.lang.matlab.MatlabTokenManager;
import net.sourceforge.pmd.lang.matlab.ast.Token; import net.sourceforge.pmd.lang.matlab.ast.Token;
import net.sourceforge.pmd.util.IOUtil; import net.sourceforge.pmd.util.IOUtil;
@ -24,17 +23,17 @@ public class MatlabTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer(); StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new MatlabTokenManager(reader)); MatlabTokenManager tokenManager = new MatlabTokenManager(reader);
tokenManager.setFileName(sourceCode.getFileName());
final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
Token currentToken = (Token) tokenFilter.getNextToken(); Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) { while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken(); currentToken = (Token) tokenFilter.getNextToken();
} }
tokenEntries.add(TokenEntry.getEOF()); } catch (IOException err) {
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError | IOException err) {
err.printStackTrace(); err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); } finally {
tokenEntries.add(TokenEntry.getEOF()); tokenEntries.add(TokenEntry.getEOF());
} }
} }

13
pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
View File

@ -10,7 +10,6 @@ import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.objectivec.ObjectiveCTokenManager; import net.sourceforge.pmd.lang.objectivec.ObjectiveCTokenManager;
import net.sourceforge.pmd.lang.objectivec.ast.Token; import net.sourceforge.pmd.lang.objectivec.ast.Token;
@ -23,20 +22,18 @@ public class ObjectiveCTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer(); StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = new StringReader(buffer.toString())) { try (Reader reader = new StringReader(buffer.toString())) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new ObjectiveCTokenManager(reader)); ObjectiveCTokenManager tokenManager = new ObjectiveCTokenManager(reader);
tokenManager.setFileName(sourceCode.getFileName());
final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
Token currentToken = (Token) tokenFilter.getNextToken(); Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) { while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken(); currentToken = (Token) tokenFilter.getNextToken();
} }
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally {
tokenEntries.add(TokenEntry.getEOF());
} }
} }
} }

17
pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
View File

@ -11,7 +11,10 @@ import java.util.logging.Logger;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager; import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.plsql.PLSQLLanguageModule;
import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants; import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants;
import net.sourceforge.pmd.lang.plsql.ast.Token; import net.sourceforge.pmd.lang.plsql.ast.Token;
@ -72,9 +75,8 @@ public class PLSQLTokenizer implements Tokenizer {
} }
String fileName = sourceCode.getFileName(); String fileName = sourceCode.getFileName();
StringBuilder sb = sourceCode.getCodeBuffer();
TokenFilter tokenFilter = new JavaCCTokenFilter(new PLSQLTokenManager(new StringReader(sb.toString()))); TokenFilter tokenFilter = createTokenFilter(sourceCode);
Token currentToken = (Token) tokenFilter.getNextToken(); Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) { while (currentToken != null) {
String image = currentToken.image; String image = currentToken.image;
@ -105,4 +107,13 @@ public class PLSQLTokenizer implements Tokenizer {
} }
} }
private JavaCCTokenFilter createTokenFilter(final SourceCode sourceCode) {
final StringBuilder stringBuilder = sourceCode.getCodeBuffer();
final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PLSQLLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString()));
return new JavaCCTokenFilter(tokenMgr);
}
} }

20
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -10,10 +10,7 @@ import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.python.PythonTokenManager;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import net.sourceforge.pmd.lang.python.ast.Token; import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.util.IOUtil; import net.sourceforge.pmd.util.IOUtil;
@ -26,21 +23,18 @@ public class PythonTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer(); StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME) PythonTokenManager tokenManager = new PythonTokenManager(reader);
.getDefaultVersion().getLanguageVersionHandler(); tokenManager.setFileName(sourceCode.getFileName());
TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions()) TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
.getTokenManager(sourceCode.getFileName(), reader));
Token currentToken = (Token) tokenFilter.getNextToken(); Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) { while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken(); currentToken = (Token) tokenFilter.getNextToken();
} }
tokenEntries.add(TokenEntry.getEOF()); } catch (IOException err) {
System.err.println("Added " + sourceCode);
} catch (TokenMgrError | IOException err) {
err.printStackTrace(); err.printStackTrace();
System.err.println("Skipping " + sourceCode + " due to parse error"); } finally {
tokenEntries.add(TokenEntry.getEOF()); tokenEntries.add(TokenEntry.getEOF());
} }
} }