Finish cpp module

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java

@@ -5,16 +5,13 @@
 package net.sourceforge.pmd.cpd;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
 
-import java.io.IOException;
 import java.util.Properties;
 
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 
@@ -39,10 +36,15 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
         return tok;
     }
 
+    @Override
+    public Properties defaultProperties() {
+        return dontSkipBlocks();
+    }
+
     @Test
     public void testUTFwithBOM() {
-        Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
-        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
+        Tokenizer tokenizer = newTokenizer(dontSkipBlocks());
+        Tokens tokens = tokenize(tokenizer, "\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
         assertEquals(15, tokens.size());
     }
 
@@ -104,8 +106,8 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
     }
 
     @Test
-    public void testEOLCommentInPreprocessingDirective() {
-        parse("#define LSTFVLES_CPP  //*" + PMD.EOL);
+    public void testPreprocessingDirectives() {
+        doTest("preprocessorDirectives");
     }
 
     @Test
@@ -128,30 +130,6 @@ public class CPPTokenizerTest extends CpdTextComparisonTest {
     }
 
 
-    private Tokens parse(String snippet) {
-        try {
-            return parse(snippet, false, new Tokens());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) throws IOException {
-        return parse(snippet, skipBlocks, null, tokens);
-    }
-
-    private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) throws IOException {
-
-        Properties properties = properties(skipBlocks, skipPattern);
-
-        CPPTokenizer tokenizer = new CPPTokenizer();
-        tokenizer.setProperties(properties);
-
-        SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
-        tokenizer.tokenize(code, tokens);
-        return tokens;
-    }
-
     private static Properties skipBlocks(String skipPattern) {
         return properties(true, skipPattern);
     }

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/preprocessorDirectives.cpp

@@ -0,0 +1,2 @@
+#define LSTFVLES_CPP  //*
+// EOL comment in the directive

pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/preprocessorDirectives.txt

@@ -0,0 +1,2 @@
+    [Image] or [Truncated image[            Bcol      Ecol
+EOF

pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java

@@ -6,7 +6,6 @@ package net.sourceforge.pmd.cpd;
 
 import java.util.Properties;
 
-import org.jetbrains.annotations.NotNull;
 import org.junit.Test;
 
 import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
@@ -17,13 +16,11 @@ public class EcmascriptTokenizerTest extends CpdTextComparisonTest {
         super(".js");
     }
 
-    @NotNull
     @Override
     public Tokenizer newTokenizer(Properties properties) {
         return new EcmascriptTokenizer();
     }
 
-    @NotNull
     @Override
     protected String getResourcePrefix() {
         return "../lang/ecmascript/cpd/testdata";

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt

@@ -53,8 +53,10 @@ abstract class CpdTextComparisonTest(
         }
     }
 
+    open fun defaultProperties() = Properties()
+
     @JvmOverloads
-    fun doTest(fileBaseName: String, expectedSuffix: String = "", properties: Properties = Properties()) {
+    fun doTest(fileBaseName: String, expectedSuffix: String = "", properties: Properties = defaultProperties()) {
         super.doTest(fileBaseName, expectedSuffix) { sourceText ->
             val sourceCode = SourceCode(SourceCode.StringCodeLoader(sourceText, "$fileBaseName$extensionIncludingDot"))
             val tokens = Tokens().also {
@@ -117,6 +119,14 @@ abstract class CpdTextComparisonTest(
 
     }
 
+
+    fun sourceCodeOf(str: String): SourceCode = SourceCode(SourceCode.StringCodeLoader(str))
+
+    fun tokenize(tokenizer: Tokenizer, str: String): Tokens =
+            Tokens().also {
+                tokenizer.tokenize(sourceCodeOf(str), it)
+            }
+
     private companion object {
         const val Indent = "    "
         const val Col0Width = 40