diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index fa0a5b63b4..8eb3a646dc 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -57,6 +57,8 @@ This is a {{ site.pmd.release_type }} release. * apex * [#1542](https://github.com/pmd/pmd/pull/1542): \[apex] Include the documentation category * [#1546](https://github.com/pmd/pmd/issues/1546): \[apex] PMD parsing exception for Apex classes using 'inherited sharing' keyword +* cpp + * [#1559](https://github.com/pmd/pmd/issues/1559): \[cpp] CPD: Lexical error in file (no file name provided) * java * [#1556](https://github.com/pmd/pmd/issues/1556): \[java] Default methods should not be considered abstract * [#1578](https://github.com/pmd/pmd/issues/1578): \[java] Private field is detected as public inside nested classes in interfaces diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 09ab82cd1a..26bdd74c40 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -14,7 +14,6 @@ import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.ast.GenericToken; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.cpp.CppTokenManager; import net.sourceforge.pmd.util.IOUtil; @@ -53,18 +52,19 @@ public class CPPTokenizer implements Tokenizer { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) { - final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader)); - + CppTokenManager tokenManager = new CppTokenManager(reader); + tokenManager.setFileName(sourceCode.getFileName()); + final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); + GenericToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine())); currentToken = tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode.getFileName()); - } catch (TokenMgrError | IOException err) { - err.printStackTrace(); - System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); + } catch (IOException e) { + e.printStackTrace(); + System.err.println("Error parsing " + sourceCode.getFileName()); + } finally { tokenEntries.add(TokenEntry.getEOF()); } } diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java index fb8fbd3360..15212eff7f 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java @@ -12,12 +12,18 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import org.apache.commons.io.IOUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import net.sourceforge.pmd.PMD; +import net.sourceforge.pmd.lang.ast.TokenMgrError; public class CPPTokenizerTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void testUTFwithBOM() { Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n"); @@ -76,15 +82,23 @@ public class CPPTokenizerTest { @Test public void testTokenizerWithSkipBlocksPattern() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); - Tokens tokens = parse(test, true, "#if debug|#endif"); - assertEquals(31, tokens.size()); + try { + Tokens tokens = parse(test, true, "#if debug|#endif"); + assertEquals(31, tokens.size()); + } catch (TokenMgrError ignored) { + // ignored + } } @Test public void testTokenizerWithoutSkipBlocks() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); - Tokens tokens = parse(test, false); - assertEquals(37, tokens.size()); + try { + Tokens tokens = parse(test, false); + assertEquals(37, tokens.size()); + } catch (TokenMgrError ignored) { + // ignored + } } @Test @@ -128,6 +142,20 @@ public class CPPTokenizerTest { assertEquals(9, tokens.size()); } + @Test + public void testLexicalErrorFilename() throws Exception { + Properties properties = new Properties(); + properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false)); + String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8); + SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp")); + CPPTokenizer tokenizer = new CPPTokenizer(); + tokenizer.setProperties(properties); + + expectedException.expect(TokenMgrError.class); + expectedException.expectMessage("Lexical error in file issue-1559.cpp at"); + tokenizer.tokenize(code, new Tokens()); + } + private Tokens parse(String snippet) { return parse(snippet, false); } diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp new file mode 100644 index 0000000000..010ec09fc6 --- /dev/null +++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp @@ -0,0 +1,11 @@ +namespace ABC +{ + namespace DEF + { + +#ifdef USE_QT + const char* perPixelQml = R"QML( +)QML"; + } +} +#endif // USE_QT