From c8a0bc9b13ce35c4c8157c56c19f75fb509bd145 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 17 Jun 2017 19:42:07 +0200 Subject: [PATCH] cpd: fix line number issue, add tests for apex cpd Fixes #427 --- .../net/sourceforge/pmd/cpd/ApexCpdTest.java | 47 ++++++++++++++ .../pmd/cpd/ApexTokenizerTest.java | 62 +++++++++++++++++++ .../net/sourceforge/pmd/cpd/Simple.cls | 10 +++ .../pmd/cpd/issue427/SFDCEncoder.cls | 29 +++++++++ .../pmd/cpd/issue427/SFDCEncoderConstants.cls | 26 ++++++++ .../pmd/cpd/AbstractTokenizer.java | 2 +- .../net/sourceforge/pmd/cpd/SourceCode.java | 6 +- .../net/sourceforge/pmd/cpd/TokenEntry.java | 6 ++ src/site/markdown/overview/changelog.md | 2 + 9 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexCpdTest.java create mode 100644 pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexTokenizerTest.java create mode 100644 pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/Simple.cls create mode 100644 pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoder.cls create mode 100644 pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoderConstants.cls diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexCpdTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexCpdTest.java new file mode 100644 index 0000000000..685afe63e8 --- /dev/null +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexCpdTest.java @@ -0,0 +1,47 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import org.apache.commons.io.FilenameUtils; +import org.junit.Before; +import org.junit.Test; + +public class ApexCpdTest { + private File testdir; + + @Before + public void setUp() { + String path = FilenameUtils.normalize("src/test/resources/net/sourceforge/pmd/cpd/issue427"); + testdir = new File(path); + } + + @Test + public void testIssue427() throws IOException { + CPDConfiguration configuration = new CPDConfiguration(); + configuration.setMinimumTileSize(50); + configuration.setLanguage(LanguageFactory.createLanguage("apex")); + CPD cpd = new CPD(configuration); + cpd.add(new File(testdir, "SFDCEncoder.cls")); + cpd.add(new File(testdir, "SFDCEncoderConstants.cls")); + + cpd.go(); + + Iterator matches = cpd.getMatches(); + int duplications = 0; + while (matches.hasNext()) { + Match match = matches.next(); + duplications++; + assertTrue(match.getSourceCodeSlice().startsWith("/*")); + } + assertEquals(1, duplications); + } +} diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexTokenizerTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexTokenizerTest.java new file mode 100644 index 0000000000..d0cab45b95 --- /dev/null +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/cpd/ApexTokenizerTest.java @@ -0,0 +1,62 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import net.sourceforge.pmd.PMD; +import net.sourceforge.pmd.cpd.SourceCode.StringCodeLoader; + +public class ApexTokenizerTest { + + @Test + public void testTokenize() throws IOException { + Tokens tokens = tokenize(load("Simple.cls")); + if (tokens.size() != 28) { + printTokens(tokens); + } + assertEquals(28, tokens.size()); + assertEquals("someparam", findTokensByLine(8, tokens).get(0).toString()); + } + + private List findTokensByLine(int line, Tokens tokens) { + List result = new ArrayList<>(); + for (TokenEntry entry : tokens.getTokens()) { + if (entry.getBeginLine() == line) { + result.add(entry); + } + } + if (result.isEmpty()) { + fail("Not tokens found at line " + line); + } + return result; + } + + private Tokens tokenize(String code) { + ApexTokenizer tokenizer = new ApexTokenizer(); + Tokens tokens = new Tokens(); + tokenizer.tokenize(new SourceCode(new StringCodeLoader(code)), tokens); + return tokens; + } + + private void printTokens(Tokens tokens) { + for (TokenEntry entry : tokens.getTokens()) { + System.out.printf("%02d: %s%s", entry.getBeginLine(), entry.toString(), PMD.EOL); + } + } + + private String load(String name) throws IOException { + return IOUtils.toString(ApexTokenizerTest.class.getResourceAsStream(name), StandardCharsets.UTF_8); + } +} diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/Simple.cls b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/Simple.cls new file mode 100644 index 0000000000..128248a602 --- /dev/null +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/Simple.cls @@ -0,0 +1,10 @@ +/* + * Some comment + */ +public with sharing class Simple { + public String someParam { get; set; } + + public void getInit() { + someParam = "test"; + } +} diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoder.cls b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoder.cls new file mode 100644 index 0000000000..90bf7950d8 --- /dev/null +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoder.cls @@ -0,0 +1,29 @@ +/** + * OWASP Enterprise Security API (ESAPI) + * + * This file is part of the Open Web Application Security Project (OWASP) + * Enterprise Security API (ESAPI) project. For details, please see + * http://www.owasp.org/index.php/ESAPI. + * + * Copyright (c) 2010 - Salesforce.com + * + * The Apex ESAPI implementation is published by Salesforce.com under the New BSD license. You should read and accept the + * LICENSE before you use, modify, and/or redistribute this software. + * + * @author Yoel Gluck (securecloud .at. salesforce.com) Salesforce.com + * @created 2010 + */ + +/** + * This class is a basic encoder/escaper to help prevent some XSS attacks etc. + */ +global with sharing class SFDCEncoder { + + /* TODO Yoel - all these functions should be converted into a white list aproach - I am using blacklist to be consistent with the VISUALFORCE functions */ + /* TODO Yoel - Do we need to encode ASCII/Unicode white-space/new-line characters? These used to cause some security issues in some browsers not sure if this is still the case */ + + /* Note - the order of these encoding strings is very important so we don't end up with double encoding. + Each string we search for, must not be found as a result of a previous encoded string replacement */ + /************ CLASS CODE HERE *************/ +} + diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoderConstants.cls b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoderConstants.cls new file mode 100644 index 0000000000..5226d0ec6c --- /dev/null +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/cpd/issue427/SFDCEncoderConstants.cls @@ -0,0 +1,26 @@ +/** + * OWASP Enterprise Security API (ESAPI) + * + * This file is part of the Open Web Application Security Project (OWASP) + * Enterprise Security API (ESAPI) project. For details, please see + * http://www.owasp.org/index.php/ESAPI. + * + * Copyright (c) 2010 - Salesforce.com + * + * The Apex ESAPI implementation is published by Salesforce.com under the New BSD license. You should read and accept the + * LICENSE before you use, modify, and/or redistribute this software. + * + * @author Yoel Gluck (securecloud .at. salesforce.com) Salesforce.com + * @created 2010 + */ + +/** + * Common character classes used for input validation, output encoding, verifying password strength + * CSRF token generation, generating salts, etc. I removed all the constatnts that are not used so we + * don't burn governor limits. + */ +public with sharing class SFDCEncoderConstants { + + /************ CLASS CODE HERE *************/ +} + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java index 39c63357e4..296deb8347 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java @@ -57,7 +57,7 @@ public abstract class AbstractTokenizer implements Tokenizer { // if ( CPD.debugEnable ) { // System.out.println("Token added:" + token.toString()); // } - tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber)); + tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1)); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java index 85490d2dfe..0a585f3720 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java @@ -42,7 +42,7 @@ public class SourceCode { c = code.get(); } if (c != null) { - return c.subList(startLine, endLine); + return c.subList(startLine - 1, endLine); } return load(startLine, endLine); } @@ -71,7 +71,7 @@ public class SourceCode { List lines = new ArrayList<>(linesToRead); // Skip lines until we reach the start point - for (int i = 0; i < startLine; i++) { + for (int i = 0; i < startLine - 1; i++) { reader.readLine(); } @@ -195,7 +195,7 @@ public class SourceCode { } public String getSlice(int startLine, int endLine) { - List lines = cl.getCodeSlice(startLine - 1, endLine); + List lines = cl.getCodeSlice(startLine, endLine); StringBuilder sb = new StringBuilder(); for (String line : lines) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java index b9a8302e63..178f0e3ebe 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java @@ -38,6 +38,12 @@ public class TokenEntry implements Comparable { this.tokenSrcID = "EOFMarker"; } + /** + * Creates a new token entry with the given informations. + * @param image + * @param tokenSrcID + * @param beginLine the linenumber, 1-based. + */ public TokenEntry(String image, String tokenSrcID, int beginLine) { setImage(image); this.tokenSrcID = tokenSrcID; diff --git a/src/site/markdown/overview/changelog.md b/src/site/markdown/overview/changelog.md index a9dd3f80c9..c1bf27745a 100644 --- a/src/site/markdown/overview/changelog.md +++ b/src/site/markdown/overview/changelog.md @@ -44,6 +44,8 @@ Fields using generics are still Work in Progress, but we expect to fully support * General * [#407](https://github.com/pmd/pmd/issues/407): \[web] Release date is not properly formatted * [#429](https://github.com/pmd/pmd/issues/429): \[core] Error when running PMD from folder with space +* apex + * [#427](https://github.com/pmd/pmd/issues/427): \[apex] CPD error when parsing apex code from release 5.5.3 * java * [#414](https://github.com/pmd/pmd/issues/414): \[java] Java 8 parsing problem with annotations for wildcards * [#415](https://github.com/pmd/pmd/issues/415): \[java] Parsing Error when having an Annotated Inner class