cpd: fix line number issue, add tests for apex cpd

Fixes #427

pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexCpdTest.java

@@ -0,0 +1,47 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.commons.io.FilenameUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ApexCpdTest {
+    private File testdir;
+
+    @Before
+    public void setUp() {
+        String path = FilenameUtils.normalize("src/test/resources/net/sourceforge/pmd/cpd/issue427");
+        testdir = new File(path);
+    }
+
+    @Test
+    public void testIssue427() throws IOException {
+        CPDConfiguration configuration = new CPDConfiguration();
+        configuration.setMinimumTileSize(50);
+        configuration.setLanguage(LanguageFactory.createLanguage("apex"));
+        CPD cpd = new CPD(configuration);
+        cpd.add(new File(testdir, "SFDCEncoder.cls"));
+        cpd.add(new File(testdir, "SFDCEncoderConstants.cls"));
+
+        cpd.go();
+
+        Iterator<Match> matches = cpd.getMatches();
+        int duplications = 0;
+        while (matches.hasNext()) {
+            Match match = matches.next();
+            duplications++;
+            assertTrue(match.getSourceCodeSlice().startsWith("/*"));
+        }
+        assertEquals(1, duplications);
+    }
+}

pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexTokenizerTest.java

@@ -0,0 +1,62 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.cpd.SourceCode.StringCodeLoader;
+
+public class ApexTokenizerTest {
+
+    @Test
+    public void testTokenize() throws IOException {
+        Tokens tokens = tokenize(load("Simple.cls"));
+        if (tokens.size() != 28) {
+            printTokens(tokens);
+        }
+        assertEquals(28, tokens.size());
+        assertEquals("someparam", findTokensByLine(8, tokens).get(0).toString());
+    }
+
+    private List<TokenEntry> findTokensByLine(int line, Tokens tokens) {
+        List<TokenEntry> result = new ArrayList<>();
+        for (TokenEntry entry : tokens.getTokens()) {
+            if (entry.getBeginLine() == line) {
+                result.add(entry);
+            }
+        }
+        if (result.isEmpty()) {
+            fail("Not tokens found at line " + line);
+        }
+        return result;
+    }
+
+    private Tokens tokenize(String code) {
+        ApexTokenizer tokenizer = new ApexTokenizer();
+        Tokens tokens = new Tokens();
+        tokenizer.tokenize(new SourceCode(new StringCodeLoader(code)), tokens);
+        return tokens;
+    }
+
+    private void printTokens(Tokens tokens) {
+        for (TokenEntry entry : tokens.getTokens()) {
+            System.out.printf("%02d: %s%s", entry.getBeginLine(), entry.toString(), PMD.EOL);
+        }
+    }
+
+    private String load(String name) throws IOException {
+        return IOUtils.toString(ApexTokenizerTest.class.getResourceAsStream(name), StandardCharsets.UTF_8);
+    }
+}

pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/Simple.cls

@@ -0,0 +1,10 @@
+/*
+ * Some comment
+ */
+public with sharing class Simple {
+    public String someParam { get; set; }
+
+    public void getInit() {
+        someParam = "test"; 
+    }
+}

pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoder.cls

@@ -0,0 +1,29 @@
+/**
+ * OWASP Enterprise Security API (ESAPI)
+ *
+ * This file is part of the Open Web Application Security Project (OWASP)
+ * Enterprise Security API (ESAPI) project. For details, please see
+ * <a href="http://www.owasp.org/index.php/ESAPI">http://www.owasp.org/index.php/ESAPI</a>.
+ *
+ * Copyright (c) 2010 - Salesforce.com
+ *
+ * The Apex ESAPI implementation is published by Salesforce.com under the New BSD license. You should read and accept the
+ * LICENSE before you use, modify, and/or redistribute this software.
+ *
+ * @author Yoel Gluck (securecloud .at. salesforce.com) <a href="http://www.salesforce.com">Salesforce.com</a>
+ * @created 2010
+ */
+
+/**
+ * This class is a basic encoder/escaper to help prevent some XSS attacks etc.
+ */
+global with sharing class SFDCEncoder {
+
+	/* TODO Yoel - all these functions should be converted into a white list aproach - I am using blacklist to be consistent with the VISUALFORCE functions */
+	/* TODO Yoel - Do we need to encode ASCII/Unicode white-space/new-line characters? These used to cause some security issues in some browsers not sure if this is still the case */
+
+	/* Note - the order of these encoding strings is very important so we don't end up with double encoding.
+	      Each string we search for, must not be found as a result of a previous encoded string replacement */
+	/************ CLASS CODE HERE *************/
+}
+

pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoderConstants.cls

@@ -0,0 +1,26 @@
+/**
+ * OWASP Enterprise Security API (ESAPI)
+ *
+ * This file is part of the Open Web Application Security Project (OWASP)
+ * Enterprise Security API (ESAPI) project. For details, please see
+ * <a href="http://www.owasp.org/index.php/ESAPI">http://www.owasp.org/index.php/ESAPI</a>.
+ *
+ * Copyright (c) 2010 - Salesforce.com
+ *
+ * The Apex ESAPI implementation is published by Salesforce.com under the New BSD license. You should read and accept the
+ * LICENSE before you use, modify, and/or redistribute this software.
+ *
+ * @author Yoel Gluck (securecloud .at. salesforce.com) <a href="http://www.salesforce.com">Salesforce.com</a>
+ * @created 2010
+ */
+
+/**
+ * Common character classes used for input validation, output encoding, verifying password strength
+ * CSRF token generation, generating salts, etc. I removed all the constatnts that are not used so we
+ * don't burn governor limits.
+ */
+public with sharing class SFDCEncoderConstants {
+
+	/************ CLASS CODE HERE *************/
+}
+

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java

@@ -57,7 +57,7 @@ public abstract class AbstractTokenizer implements Tokenizer {
                     // if ( CPD.debugEnable ) {
                     // System.out.println("Token added:" + token.toString());
                     // }
-                    tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber));
+                    tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1));
 
                 }
             }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java

@@ -42,7 +42,7 @@ public class SourceCode {
                 c = code.get();
             }
             if (c != null) {
-                return c.subList(startLine, endLine);
+                return c.subList(startLine - 1, endLine);
             }
             return load(startLine, endLine);
         }
@@ -71,7 +71,7 @@ public class SourceCode {
                 List<String> lines = new ArrayList<>(linesToRead);
 
                 // Skip lines until we reach the start point
-                for (int i = 0; i < startLine; i++) {
+                for (int i = 0; i < startLine - 1; i++) {
                     reader.readLine();
                 }
 
@@ -195,7 +195,7 @@ public class SourceCode {
     }
 
     public String getSlice(int startLine, int endLine) {
-        List<String> lines = cl.getCodeSlice(startLine - 1, endLine);
+        List<String> lines = cl.getCodeSlice(startLine, endLine);
 
         StringBuilder sb = new StringBuilder();
         for (String line : lines) {

pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java

@@ -38,6 +38,12 @@ public class TokenEntry implements Comparable<TokenEntry> {
         this.tokenSrcID = "EOFMarker";
     }
 
+    /**
+     * Creates a new token entry with the given informations.
+     * @param image
+     * @param tokenSrcID
+     * @param beginLine the linenumber, 1-based.
+     */
     public TokenEntry(String image, String tokenSrcID, int beginLine) {
         setImage(image);
         this.tokenSrcID = tokenSrcID;

src/site/markdown/overview/changelog.md

@@ -44,6 +44,8 @@ Fields using generics are still Work in Progress, but we expect to fully support
 *   General
     *   [#407](https://github.com/pmd/pmd/issues/407): \[web] Release date is not properly formatted
     *   [#429](https://github.com/pmd/pmd/issues/429): \[core] Error when running PMD from folder with space
+*   apex
+    *   [#427](https://github.com/pmd/pmd/issues/427): \[apex] CPD error when parsing apex code from release 5.5.3
 *   java
     *   [#414](https://github.com/pmd/pmd/issues/414): \[java] Java 8 parsing problem with annotations for wildcards
     *   [#415](https://github.com/pmd/pmd/issues/415): \[java] Parsing Error when having an Annotated Inner class