diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java index 1a8d007a09..bb8e191f4d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java @@ -25,6 +25,7 @@ public abstract class AbstractTokenizer implements Tokenizer { private String currentLine; protected boolean spanMultipleLinesString = true; // Most languages do, so default is true + protected Character spanMultipleLinesLineContinuationCharacter = null; private boolean downcaseString = true; @@ -111,10 +112,15 @@ public abstract class AbstractTokenizer implements Tokenizer { spanMultipleLinesString && // ... the language allow multiple line span Strings lineNumber < code.size() - 1 // ... there is still more lines to parse ) { + // removes last character, if it is the line continuation (e.g. backslash) character + if (spanMultipleLinesLineContinuationCharacter != null && token.length() > 0 + && token.charAt(token.length() - 1) == spanMultipleLinesLineContinuationCharacter.charValue()) { + token.deleteCharAt(token.length() - 1); + } // parsing new line currentLine = code.get(++lineNumber); // Warning : recursive call ! - loc = parseString(token, loc, stringDelimiter); + loc = parseString(token, 0, stringDelimiter); } return loc + 1; } diff --git a/pmd-dist/src/main/scripts/run.sh b/pmd-dist/src/main/scripts/run.sh index bd204b7ea8..5b7106ffa0 100755 --- a/pmd-dist/src/main/scripts/run.sh +++ b/pmd-dist/src/main/scripts/run.sh @@ -45,7 +45,7 @@ convert_cygwin_vars() { } java_heapsize_settings() { - local heapsize=${HEAPSIZE:-512m} + local heapsize=${HEAPSIZE} case "${heapsize}" in [1-9]*[mgMG]) readonly HEAPSIZE="-Xmx${heapsize}" @@ -53,7 +53,7 @@ java_heapsize_settings() { '') ;; *) - echo "HEAPSIZE '${HEAPSIZE}' unknown (try: 512m)" + echo "HEAPSIZE '${HEAPSIZE}' unknown (try: 1024m)" exit 1 esac } @@ -123,4 +123,4 @@ cygwin_paths java_heapsize_settings -java "${HEAPSIZE}" -cp "${classpath}" "${CLASSNAME}" ${@} +java ${HEAPSIZE} -cp "${classpath}" "${CLASSNAME}" ${@} diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index 75c85f7504..386ab75f82 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -19,8 +19,9 @@ public class EcmascriptTokenizer extends AbstractTokenizer { // setting markers for 'ignorable string' in javascript this.ignorableStmt = new ArrayList(); - // strings do not span multiple lines in javascript - the lines would need to end with backslashes - // - which is not supported by this tokenizer - this.spanMultipleLinesString = false; + // strings do indeed span multiple lines in javascript + this.spanMultipleLinesString = true; + // the lines do to end with backslashes + this.spanMultipleLinesLineContinuationCharacter = '\\'; } } \ No newline at end of file diff --git a/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java b/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java index d1676870e8..b848625a1d 100644 --- a/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java +++ b/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.cpd; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import java.io.IOException; import java.util.List; @@ -40,15 +41,20 @@ public class EcmascriptTokenizerTest { public void parseStringNotAsMultiline() throws IOException { Tokenizer t = new EcmascriptTokenizer(); SourceCode sourceCode = new SourceCode( new SourceCode.StringCodeLoader( - "var s = \"a string\\\n" + "var s = \"a string \\\n" + "continues\";\n" - + "var s = \"a string\\\n" + + "var s = \"a string \\\n" + "continues2\";\n") ); Tokens tokens = new Tokens(); t.tokenize(sourceCode, tokens); - assertEquals(13, tokens.size()); + assertEquals(9, tokens.size()); List list = tokens.getTokens(); - assertEquals("\"a string", list.get(3).getIdentifier(), list.get(9).getIdentifier()); + assertEquals("var", list.get(0).getIdentifier(), list.get(4).getIdentifier()); + assertEquals("s", list.get(1).getIdentifier(), list.get(5).getIdentifier()); + assertEquals("=", list.get(2).getIdentifier(), list.get(6).getIdentifier()); + assertEquals("\"a string continues\"", list.get(3).toString()); + assertEquals("\"a string continues2\"", list.get(7).toString()); + assertFalse(list.get(3).getIdentifier() == list.get(7).getIdentifier()); } // no semi-colons diff --git a/src/site/markdown/overview/changelog.md b/src/site/markdown/overview/changelog.md index a4965d9cd7..bac8821808 100644 --- a/src/site/markdown/overview/changelog.md +++ b/src/site/markdown/overview/changelog.md @@ -10,6 +10,7 @@ **Bugfixes:** +* [#1254](https://sourceforge.net/p/pmd/bugs/1254/): CPD run that worked in 5.1.2 fails in 5.1.3 with OOM * [#1276](https://sourceforge.net/p/pmd/bugs/1276/): False positive in UnusedPrivateMethod with inner enum * [#1281](https://sourceforge.net/p/pmd/bugs/1281/): UnusedPrivateMethod incorrectly flagged for methods nested private classes * [#1282](https://sourceforge.net/p/pmd/bugs/1282/): False Positive with implicit String.valuesOf() (Java)