Merge branch 'master' into 7.0.x

pmd-python/pom.xml

@@ -82,10 +82,20 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.junit.vintage</groupId>
+            <artifactId>junit-vintage-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>net.sourceforge.pmd</groupId>
             <artifactId>pmd-test</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.sourceforge.pmd</groupId>
+            <artifactId>pmd-lang-test</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java

@@ -4,64 +4,41 @@
 
 package net.sourceforge.pmd.cpd;
 
-import static org.junit.Assert.assertEquals;
+import java.util.Properties;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.Before;
 import org.junit.Test;
 
-import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
+import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
 
-public class PythonTokenizerTest extends AbstractTokenizerTest {
+public class PythonTokenizerTest extends CpdTextComparisonTest {
 
-    private static final String FILENAME = "sample-python.py";
-
-    @Before
-    @Override
-    public void buildTokenizer() throws IOException {
-        this.tokenizer = new PythonTokenizer();
-        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
+    public PythonTokenizerTest() {
+        super(".py");
     }
 
     @Override
-    public String getSampleCode() throws IOException {
-        return IOUtils.toString(PythonTokenizer.class.getResourceAsStream(FILENAME), StandardCharsets.UTF_8);
+    protected String getResourcePrefix() {
+        return "../lang/python/cpd/testdata";
+    }
+
+    @Override
+    public Tokenizer newTokenizer(Properties properties) {
+        return new PythonTokenizer();
+    }
+    
+    
+    @Test
+    public void sampleTest() {
+        doTest("sample_python");
     }
 
     @Test
-    public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 1218;
-        super.tokenizeTest();
+    public void specialComments() {
+        doTest("special_comments");
     }
 
     @Test
-    public void testIgnoreBetweenSpecialComments() throws IOException {
-        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("import logging\n"
-                + "# CPD-OFF\n"
-                + "logger = logging.getLogger('django.request')\n"
-                + "class BaseHandler(object):\n"
-                + "    def __init__(self):\n"
-                + "        self._request_middleware = None\n"
-                + "        # CPD-ON\n"
-        ));
-        Tokens tokens = new Tokens();
-        tokenizer.tokenize(sourceCode, tokens);
-        TokenEntry.getEOF();
-        assertEquals(3, tokens.size()); // 3 tokens: "import" + "logging" + EOF
-    }
-
-    @Test
-    public void testBackticks() throws IOException {
-        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("test = 'hello'\n"
-                + "quoted = `test`\n"
-                + "print quoted\n"
-        ));
-        Tokens tokens = new Tokens();
-        tokenizer.tokenize(sourceCode, tokens); // should not result in parse error
-        TokenEntry.getEOF();
-        assertEquals(3, tokens.getTokens().get(tokens.getTokens().size() - 2).getBeginLine());
+    public void testBackticks() {
+        doTest("backticks");
     }
 }

pmd-python/src/test/resources/net/sourceforge/pmd/lang/python/cpd/testdata/backticks.py

@@ -0,0 +1,3 @@
+test = 'hello'
+quoted = `test`
+print quoted

pmd-python/src/test/resources/net/sourceforge/pmd/lang/python/cpd/testdata/backticks.txt

@@ -0,0 +1,15 @@
+    [Image] or [Truncated image[            Bcol      Ecol
+L1
+    [test]                                  1         4
+    [=]                                     6         6
+    ['hello']                               8         14
+L2
+    [quoted]                                1         6
+    [=]                                     8         8
+    [`]                                     10        10
+    [test]                                  11        14
+    [`]                                     15        15
+L3
+    [print]                                 1         5
+    [quoted]                                7         12
+EOF

pmd-python/src/test/resources/net/sourceforge/pmd/lang/python/cpd/testdata/special_comments.py

@@ -0,0 +1,7 @@
+import logging
+# CPD-OFF
+logger = logging.getLogger('django.request')
+class BaseHandler(object):
+    def __init__(self):
+        self._request_middleware = None
+        # CPD-ON

pmd-python/src/test/resources/net/sourceforge/pmd/lang/python/cpd/testdata/special_comments.txt

@@ -0,0 +1,5 @@
+    [Image] or [Truncated image[            Bcol      Ecol
+L1
+    [import]                                1         6
+    [logging]                               8         14
+EOF