Add tests

pmd-cpp/etc/grammar/cpp.jj

@@ -305,7 +305,7 @@ TOKEN :
 {
   < #CHRPREF : <STRPREF>>
 | <  CHARACTER : <CHRPREF>
-      "'" ( ( ~["'","\\","\r","\n"] ) | ( "\\" ( ~["\n","\r"] ) ) )* "'" >
+      "'" ( ( ~["'","\\","\r","\n"] ) | ( "\\" ( ~["\n","\r"] ) ) )+ "'" >
 
 | < #STRPREF : ("L" | "u" | "U" | "u8")? >
 | <  STRING : <STRPREF>

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java

@@ -8,18 +8,12 @@ import static org.junit.Assert.assertEquals;
 
 import java.util.Properties;
 
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
 public class CPPTokenizerTest extends CpdTextComparisonTest {
 
-    @Rule
-    public ExpectedException expectedException = ExpectedException.none();
-
     public CPPTokenizerTest() {
         super(".cpp");
     }
@@ -73,12 +67,6 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
         doTest("specialComments");
     }
 
-    @Test
-    public void testUnicodeEscapeInIdentifier() {
-        Tokens tokens = parse(" void main() { int a\\u0048; }");
-        assertEquals(10, tokens.size());
-    }
-
     @Test
     public void testMultiLineMacros() {
         doTest("multilineMacros");
@@ -94,7 +82,6 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
         expectTokenMgrError(" void main() { int ⚜ = __; }");
     }
 
-
     @Test
     public void testTokenizerWithSkipBlocks() {
         doTest("simpleSkipBlocks", "_skipDefault", skipBlocks());
@@ -128,10 +115,7 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
 
     @Test
     public void testLexicalErrorFilename() {
-        expectedException.expect(TokenMgrError.class);
+        expectTokenMgrError(sourceText("issue-1559"), dontSkipBlocks());
-        expectedException.expectMessage("Lexical error in file issue-1559.cpp at");
-
-        doTest("issue-1559", "", dontSkipBlocks());
     }
 
 

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/identifierChars.cpp

@@ -3,4 +3,5 @@
     int $yx = 42;
     int 県 = µweiß42;
     int ❶ = __;
+    int a\u0048; // unicode escape
  }

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/identifierChars.txt

@@ -30,5 +30,9 @@ L5
     [__]                                    13        14
     [;]                                     15        15
 L6
+    [int]                                   5         7
+    [a\\u0048]                              9         15
+    [;]                                     16        16
+L7
     [}]                                     2         2
 EOF

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/literals.cpp

@@ -1,7 +1,7 @@
  void main() {
     char x = L'a'; // wide chars
     x = '\0x05';   // hex
-    x = L'';       // empty
+    // x = L'';    // empty character is an error
 
     print("\    oMedia"); // whitespace escape
 
@@ -32,7 +32,7 @@
 
 
     // digit separators
-    auto integer_literal = 1'000'000;
+    auto integer_literal = 1'000''000;
     auto floating_point_literal = 0.000'015'3;
     auto hex_literal = 0x0F00'abcd'6f3d;
     auto silly_example = 1'0'0'000'00;

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/literals.txt

@@ -16,11 +16,6 @@ L3
     [=]                                     7         7
     ['\\0x05']                              9         15
     [;]                                     16        16
-L4
-    [x]                                     5         5
-    [=]                                     7         7
-    [L'']                                   9         11
-    [;]                                     12        12
 L6
     [print]                                 5         9
     [(]                                     10        10
@@ -103,8 +98,8 @@ L35
     [auto]                                  5         8
     [integer_literal]                       10        24
     [=]                                     26        26
-    [1'000'000]                             28        36
+    [1'000''000]                            28        37
-    [;]                                     37        37
+    [;]                                     38        38
 L36
     [auto]                                  5         8
     [floating_point_literal]                10        31

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/test/BaseTextComparisonTest.kt

@@ -39,14 +39,8 @@ abstract class BaseTextComparisonTest {
                         expectedSuffix: String = "",
                         transformTextContent: (String) -> String) {
         val expectedFile = findTestFile(resourceLoader, "${resourcePrefix}/$fileBaseName$expectedSuffix$ExpectedExt").toFile()
-        val sourceFile = findTestFile(resourceLoader, "${resourcePrefix}/$fileBaseName$extensionIncludingDot").toFile()
 
-        assert(sourceFile.isFile) {
+        val actual = transformTextContent(sourceText(fileBaseName))
-            "Source file $sourceFile is missing"
-        }
-
-        val sourceText = sourceFile.readText(Charsets.UTF_8).normalize()
-        val actual = transformTextContent(sourceText)
 
         if (!expectedFile.exists()) {
             expectedFile.writeText(actual)
@@ -58,6 +52,17 @@ abstract class BaseTextComparisonTest {
         assertEquals(expected.normalize(), actual.normalize(), "File comparison failed, see the reference: $expectedFile")
     }
 
+    protected fun sourceText(fileBaseName: String): String {
+        val sourceFile = findTestFile(resourceLoader, "${resourcePrefix}/$fileBaseName$extensionIncludingDot").toFile()
+
+        assert(sourceFile.isFile) {
+            "Source file $sourceFile is missing"
+        }
+
+        val sourceText = sourceFile.readText(Charsets.UTF_8).normalize()
+        return sourceText
+    }
+
     // Outputting a path makes for better error messages
     private val srcTestResources = let {
         // this is set from maven surefire