#1254 CPD run that worked in 5.1.2 fails in 5.1.3 with OOM

Removed default heapsize of 512m
reenabled multiple line strings in javascript

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java

@@ -25,6 +25,7 @@ public abstract class AbstractTokenizer implements Tokenizer {
 	private String currentLine;
 
 	protected boolean spanMultipleLinesString = true;	// Most languages do, so default is true
+	protected Character spanMultipleLinesLineContinuationCharacter = null;
 
 	private boolean downcaseString = true;
 
@@ -111,10 +112,15 @@ public abstract class AbstractTokenizer implements Tokenizer {
         		spanMultipleLinesString && // ... the language allow multiple line span Strings
         		lineNumber < code.size() - 1 // ... there is still more lines to parse
         	) {
+            // removes last character, if it is the line continuation (e.g. backslash) character
+            if (spanMultipleLinesLineContinuationCharacter != null && token.length() > 0
+                    && token.charAt(token.length() - 1) == spanMultipleLinesLineContinuationCharacter.charValue()) {
+                token.deleteCharAt(token.length() - 1);
+            }
         	// parsing new line
         	currentLine = code.get(++lineNumber);
         	// Warning : recursive call !
-        	loc = parseString(token, loc, stringDelimiter);
+        	loc = parseString(token, 0, stringDelimiter);
         }
         return loc + 1;
     }

pmd-dist/src/main/scripts/run.sh

@@ -45,7 +45,7 @@ convert_cygwin_vars() {
 }
 
 java_heapsize_settings() {
-    local heapsize=${HEAPSIZE:-512m}
+    local heapsize=${HEAPSIZE}
     case "${heapsize}" in
         [1-9]*[mgMG])
             readonly HEAPSIZE="-Xmx${heapsize}"
@@ -53,7 +53,7 @@ java_heapsize_settings() {
         '')
             ;;
         *)
-            echo "HEAPSIZE '${HEAPSIZE}' unknown (try: 512m)"
+            echo "HEAPSIZE '${HEAPSIZE}' unknown (try: 1024m)"
             exit 1
     esac
 }
@@ -123,4 +123,4 @@ cygwin_paths
 
 java_heapsize_settings
 
-java "${HEAPSIZE}" -cp "${classpath}" "${CLASSNAME}" ${@}
+java ${HEAPSIZE} -cp "${classpath}" "${CLASSNAME}" ${@}

pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java

@@ -19,8 +19,9 @@ public class EcmascriptTokenizer extends AbstractTokenizer {
         // setting markers for 'ignorable string' in javascript
         this.ignorableStmt = new ArrayList<String>();
 
-        // strings do not span multiple lines in javascript - the lines would need to end with backslashes
+        // strings do indeed span multiple lines in javascript
-        // - which is not supported by this tokenizer
+        this.spanMultipleLinesString = true;
-        this.spanMultipleLinesString = false;
+        // the lines do to end with backslashes
+        this.spanMultipleLinesLineContinuationCharacter = '\\';
     }
 }

pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java

@@ -4,6 +4,7 @@
 package net.sourceforge.pmd.cpd;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,15 +41,20 @@ public class EcmascriptTokenizerTest {
     public void parseStringNotAsMultiline() throws IOException {
         Tokenizer t = new EcmascriptTokenizer();
         SourceCode sourceCode = new SourceCode( new SourceCode.StringCodeLoader(
-                  "var s = \"a string\\\n"
+                  "var s = \"a string \\\n"
                 + "continues\";\n"
-                + "var s = \"a string\\\n"
+                + "var s = \"a string \\\n"
                 + "continues2\";\n") );
         Tokens tokens = new Tokens();
         t.tokenize(sourceCode, tokens);
-        assertEquals(13, tokens.size());
+        assertEquals(9, tokens.size());
         List<TokenEntry> list = tokens.getTokens();
-        assertEquals("\"a string", list.get(3).getIdentifier(), list.get(9).getIdentifier());
+        assertEquals("var", list.get(0).getIdentifier(), list.get(4).getIdentifier());
+        assertEquals("s", list.get(1).getIdentifier(), list.get(5).getIdentifier());
+        assertEquals("=", list.get(2).getIdentifier(), list.get(6).getIdentifier());
+        assertEquals("\"a string continues\"", list.get(3).toString());
+        assertEquals("\"a string continues2\"", list.get(7).toString());
+        assertFalse(list.get(3).getIdentifier() == list.get(7).getIdentifier());
     }
 
     // no semi-colons

src/site/markdown/overview/changelog.md

@@ -10,6 +10,7 @@
 
 **Bugfixes:**
 
+* [#1254](https://sourceforge.net/p/pmd/bugs/1254/): CPD run that worked in 5.1.2 fails in 5.1.3 with OOM
 * [#1276](https://sourceforge.net/p/pmd/bugs/1276/): False positive in UnusedPrivateMethod with inner enum
 * [#1281](https://sourceforge.net/p/pmd/bugs/1281/): UnusedPrivateMethod incorrectly flagged for methods nested private classes
 * [#1282](https://sourceforge.net/p/pmd/bugs/1282/): False Positive with implicit String.valuesOf() (Java)