Merge branch 'pr-490'

pmd-cpp/etc/grammar/cpp.jj

@@ -149,10 +149,6 @@ SKIP :
   "/*"  : IN_COMMENT
 |
   "#" : PREPROCESSOR_OUTPUT
-|
-  "\\\r\n"
-|
-  "\\\n"
 }
 
 <IN_LINE_COMMENT> SKIP:
@@ -325,7 +321,7 @@ TOKEN :
 
   <  CHARACTER : ("L")? "'" ( ( ~["'","\\","\r","\n"] ) | ( "\\" ( ~["\n","\r"] ) ) )* "'" >
 
-| <  STRING : ("L")? "\"" ( ( ~["\"","\\","\r","\n"] ) | ( ("\\")+ ( ~["\n","\r"] | "\n" | "\r\n" ) ) )* "\"" >
+| <  STRING : ("L")? "\"" ( ( ~["\"","\\","\r","\n"] ) | ( "\\" ( ~["\n","\r"] | "\n" | "\r\n" ) ) )* "\"" >
 
 | <  RSTRING : "R\"(" ( ~[")"] | ( ")" ~["\""] ) )* ")\"" >
 }

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java

@@ -0,0 +1,62 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.cpp;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.regex.Pattern;
+
+import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+
+/**
+ * A SimpleCharStream, that supports the continuation of lines via backslash+newline,
+ * which is used in C/C++.
+ *
+ * @author Andreas Dangel
+ */
+public class CppCharStream extends SimpleCharStream {
+
+    private static final Pattern CONTINUATION = Pattern.compile("\\\\\\n|\\\\\\r\\n");
+    private static final char BACKSLASH = '\\';
+    private static final char NEWLINE = '\n';
+    private static final char CARRIAGE_RETURN = '\r';
+
+    public CppCharStream(Reader dstream) {
+        super(dstream);
+    }
+
+    @Override
+    public char readChar() throws IOException {
+        char c = super.readChar();
+        if (c == BACKSLASH) {
+            char c1 = super.readChar();
+            if (c1 == NEWLINE) {
+                c = super.readChar();
+            } else if (c1 == CARRIAGE_RETURN) {
+                char c2 = super.readChar();
+                if (c2 == NEWLINE) {
+                    c = super.readChar();
+                } else {
+                    backup(2);
+                }
+            } else {
+                backup(1);
+            }
+        }
+        return c;
+    }
+
+    @Override
+    public char[] GetSuffix(int len) {
+        String image = GetImage();
+        return image.substring(image.length() - len, image.length()).toCharArray();
+    }
+
+    @Override
+    public String GetImage() {
+        String image = super.GetImage();
+        return CONTINUATION.matcher(image).replaceAll("");
+    }
+}

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppTokenManager.java

@@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.cpp;
 import java.io.Reader;
 
 import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
 import net.sourceforge.pmd.lang.cpp.ast.CppParserTokenManager;
 
 /**
@@ -23,7 +22,7 @@ public class CppTokenManager implements TokenManager {
      *            the source code
      */
     public CppTokenManager(Reader source) {
-        tokenManager = new CppParserTokenManager(new SimpleCharStream(source));
+        tokenManager = new CppParserTokenManager(new CppCharStream(source));
     }
 
     public Object getNextToken() {

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java

@@ -7,17 +7,19 @@ package net.sourceforge.pmd.cpd;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
+import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
 import org.apache.commons.io.IOUtils;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.SourceCode.StringCodeLoader;
+import net.sourceforge.pmd.lang.cpp.CppTokenManager;
+import net.sourceforge.pmd.lang.cpp.ast.Token;
 
 public class CPPTokenizerContinuationTest {
 
@@ -25,32 +27,61 @@ public class CPPTokenizerContinuationTest {
     public void parseWithContinuation() throws Exception {
         String code = load("cpp_with_continuation.cpp");
         Tokens tokens = parse(code);
-        if (tokens.size() < 53) {
+        if (tokens.size() < 52) {
             printTokens(tokens);
-            fail("Not enough tokens - probably parsing error");
+            fail("Not enough tokens - probably parsing error. Tokens: " + tokens.size());
         }
 
         assertEquals("static", findByLine(8, tokens).get(0).toString());
         assertEquals("int", findByLine(8, tokens).get(1).toString());
 
-        // Note: the token should be "ab", but since we skip "\\\n" between a and b,
-        // we end up with two tokens. -> the tokens are not joined.
-        // This could lead to false negatives in duplication detection.
-        // However, this is only a problem, if the continuation character is added *within*
-        // a token and not at token boundaries.
+        // special case, if the continuation is *within* a token
         // see also test #testContinuationIntraToken
-        //assertEquals("ab", findByLine(8, tokens).get(2).toString());
+        TokenEntry tokenEntry = findByLine(8, tokens).get(2);
+        assertEquals("ab", tokenEntry.toString());
 
         assertEquals("int", findByLine(12, tokens).get(0).toString());
         assertEquals("main", findByLine(12, tokens).get(1).toString());
         assertEquals("(", findByLine(12, tokens).get(2).toString());
         assertEquals(")", findByLine(12, tokens).get(3).toString());
         assertEquals("{", findByLine(13, tokens).get(0).toString());
+        assertEquals("\"world!\\n\"", findByLine(16, tokens).get(0).toString());
+        assertEquals("\"3 Hello, \\world!\\n\"", findByLine(22, tokens).get(4).toString());
         assertEquals("}", findByLine(29, tokens).get(0).toString());
     }
 
+    /**
+     * Verifies the begin/end of a token. Uses the underlaying JavaCC Token and
+     * not TokenEntry.
+     */
+    @Test
+    public void parseWithContinuationCppTokenManager() throws Exception {
+        String code = load("cpp_with_continuation.cpp");
+        CppTokenManager tokenManager = new CppTokenManager(new StringReader(code));
+        List<Token> tokens = new ArrayList<>();
+
+        Token token = (Token) tokenManager.getNextToken();
+        while (!token.image.isEmpty()) {
+            tokens.add(token);
+            token = (Token) tokenManager.getNextToken();
+        }
+
+        assertEquals(51, tokens.size());
+
+        assertToken(tokens.get(2), "ab", 8, 12, 9, 1);
+        assertToken(tokens.get(22), "\"2 Hello, world!\\n\"", 18, 16, 19, 9);
+    }
+
+
+    private void assertToken(Token token, String image, int beginLine, int beginColumn, int endLine, int endColumn) {
+        assertEquals(image, token.image);
+        assertEquals(beginLine, token.beginLine);
+        assertEquals(beginColumn, token.beginColumn);
+        assertEquals(endLine, token.endLine);
+        assertEquals(endColumn, token.endColumn);
+    }
+
     @Test
-    @Ignore
     public void testContinuationIntraToken() throws Exception {
         Tokens tokens = parse(load("cpp_continuation_intra_token.cpp"));
         assertEquals(7, tokens.size());

pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/CppCharStreamTest.java

@@ -0,0 +1,44 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.cpp;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.junit.Test;
+
+public class CppCharStreamTest {
+
+    @Test
+    public void testContinuationUnix() throws IOException {
+        CppCharStream stream = new CppCharStream(new StringReader("a\\\nb"));
+        assertStream(stream, "ab");
+    }
+
+    @Test
+    public void testContinuationWindows() throws IOException {
+        CppCharStream stream = new CppCharStream(new StringReader("a\\\r\nb"));
+        assertStream(stream, "ab");
+    }
+
+    @Test
+    public void testBackup() throws IOException {
+        CppCharStream stream = new CppCharStream(new StringReader("a\\b\\\rc"));
+        assertStream(stream, "a\\b\\\rc");
+    }
+
+    private void assertStream(CppCharStream stream, String token) throws IOException {
+        char c = stream.BeginToken();
+        assertEquals(token.charAt(0), c);
+        for (int i = 1; i < token.length(); i++) {
+            c = stream.readChar();
+            assertEquals(token.charAt(i), c);
+        }
+        assertEquals(token, stream.GetImage());
+        assertEquals(token, new String(stream.GetSuffix(token.length())));
+    }
+}

src/site/markdown/overview/changelog.md

@@ -35,6 +35,8 @@ CPD will therefore have less false positives and false negatives.
 
 *   apex
     *   [#488](https://github.com/pmd/pmd/pull/488): \[apex] Use Apex lexer for CPD
+*   cpp
+    *   [#448](https://github.com/pmd/pmd/issues/448): \[cpp] Write custom CharStream to handle continuation characters
 *   java
     *   [#1513](https://sourceforge.net/p/pmd/bugs/1513/): \[java] Remove deprecated rule UseSingleton
 *   java-controversial