[cpp] CPD: Lexical error in file (no file name provided)

Fixes #1559

docs/pages/release_notes.md

@@ -57,6 +57,8 @@ This is a {{ site.pmd.release_type }} release.
 *   apex
     *   [#1542](https://github.com/pmd/pmd/pull/1542): \[apex] Include the documentation category
     *   [#1546](https://github.com/pmd/pmd/issues/1546): \[apex] PMD parsing exception for Apex classes using 'inherited sharing' keyword
+*   cpp
+    *   [#1559](https://github.com/pmd/pmd/issues/1559): \[cpp] CPD: Lexical error in file (no file name provided)
 *   java
     *   [#1556](https://github.com/pmd/pmd/issues/1556): \[java] Default methods should not be considered abstract
     *   [#1578](https://github.com/pmd/pmd/issues/1578): \[java] Private field is detected as public inside nested classes in interfaces

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -14,7 +14,6 @@ import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.ast.GenericToken;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.cpp.CppTokenManager;
 import net.sourceforge.pmd.util.IOUtil;
 
@@ -53,18 +52,19 @@ public class CPPTokenizer implements Tokenizer {
     public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
         StringBuilder buffer = sourceCode.getCodeBuffer();
         try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) {
-            final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader));
-            
+            CppTokenManager tokenManager = new CppTokenManager(reader);
+            tokenManager.setFileName(sourceCode.getFileName());
+            final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
+
             GenericToken currentToken = tokenFilter.getNextToken();
             while (currentToken != null) {
                 tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
                 currentToken = tokenFilter.getNextToken();
             }
-            tokenEntries.add(TokenEntry.getEOF());
-            System.err.println("Added " + sourceCode.getFileName());
-        } catch (TokenMgrError | IOException err) {
-            err.printStackTrace();
-            System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
+        } catch (IOException e) {
+            e.printStackTrace();
+            System.err.println("Error parsing " + sourceCode.getFileName());
+        } finally {
             tokenEntries.add(TokenEntry.getEOF());
         }
     }

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java

@@ -12,12 +12,18 @@ import java.nio.charset.StandardCharsets;
 import java.util.Properties;
 
 import org.apache.commons.io.IOUtils;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 public class CPPTokenizerTest {
 
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
     @Test
     public void testUTFwithBOM() {
         Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
@@ -76,15 +82,23 @@ public class CPPTokenizerTest {
     @Test
     public void testTokenizerWithSkipBlocksPattern() throws Exception {
         String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
-        Tokens tokens = parse(test, true, "#if debug|#endif");
-        assertEquals(31, tokens.size());
+        try {
+            Tokens tokens = parse(test, true, "#if debug|#endif");
+            assertEquals(31, tokens.size());
+        } catch (TokenMgrError ignored) {
+            // ignored
+        }
     }
 
     @Test
     public void testTokenizerWithoutSkipBlocks() throws Exception {
         String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
-        Tokens tokens = parse(test, false);
-        assertEquals(37, tokens.size());
+        try {
+            Tokens tokens = parse(test, false);
+            assertEquals(37, tokens.size());
+        } catch (TokenMgrError ignored) {
+            // ignored
+        }
     }
 
     @Test
@@ -128,6 +142,20 @@ public class CPPTokenizerTest {
         assertEquals(9, tokens.size());
     }
 
+    @Test
+    public void testLexicalErrorFilename() throws Exception {
+        Properties properties = new Properties();
+        properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false));
+        String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8);
+        SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp"));
+        CPPTokenizer tokenizer = new CPPTokenizer();
+        tokenizer.setProperties(properties);
+
+        expectedException.expect(TokenMgrError.class);
+        expectedException.expectMessage("Lexical error in file issue-1559.cpp at");
+        tokenizer.tokenize(code, new Tokens());
+    }
+
     private Tokens parse(String snippet) {
         return parse(snippet, false);
     }

pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp

@@ -0,0 +1,11 @@
+namespace ABC
+{
+  namespace DEF
+  {
+
+#ifdef USE_QT
+    const char* perPixelQml = R"QML(
+)QML";
+  }
+}
+#endif // USE_QT