[cpp] Allow CPD suppressions with comments

pmd-cpp/etc/grammar/cpp.jj

@@ -129,10 +129,9 @@ public final class CppParser {
       return sym.IsCtor(GetFullyScopedName());
   }
 }
-
 PARSER_END(CppParser)
 
-SKIP :
+SKIP:
 {
   " "
 |
@@ -143,38 +142,28 @@ SKIP :
   "\r\n"
 |
   "\n"
-|
-  "//" : IN_LINE_COMMENT
-|
-  "/*"  : IN_COMMENT
 |
   "#" : PREPROCESSOR_OUTPUT
 }
 
-<IN_LINE_COMMENT> SKIP:
-{
-   "\n" : DEFAULT
-}
+<DEFAULT,PREPROCESSOR_OUTPUT> SPECIAL_TOKEN:
+{ <SINGLE_LINE_COMMENT: "//"(~["\n","\r"])* ("\n"|"\r"|"\r\n")?> }
 
-<IN_LINE_COMMENT> MORE:
-{
-  < ~[] >
-}
+MORE:
+{ "/*" : IN_MULTI_LINE_COMMENT }
 
-<IN_COMMENT> SKIP:
-{ "*/" : DEFAULT }
+<IN_MULTI_LINE_COMMENT> SPECIAL_TOKEN:
+{ <MULTI_LINE_COMMENT: "*/">: DEFAULT }
 
-<IN_COMMENT,IN_PREPROCESSOR_OUTPUT_COMMENT> MORE:
+<IN_MULTI_LINE_COMMENT,IN_PREPROCESSOR_OUTPUT_COMMENT> MORE:
 { < ~[] > }
 
-<IN_PREPROCESSOR_OUTPUT_COMMENT> SKIP:
-{ "*/" : PREPROCESSOR_OUTPUT }
+<IN_PREPROCESSOR_OUTPUT_COMMENT> SPECIAL_TOKEN:
+{ <PREPROCESSOR_OUTPUT_COMMENT: "*/">: PREPROCESSOR_OUTPUT }
 
 <PREPROCESSOR_OUTPUT> SKIP:
 {
    "\n" : DEFAULT
-   | "/*" : IN_PREPROCESSOR_OUTPUT_COMMENT
-   | "//" : IN_LINE_COMMENT
 }
 
 <PREPROCESSOR_OUTPUT> MORE:
@@ -183,6 +172,8 @@ SKIP :
   |
   "\\\r\n"
   |
+  "/*": IN_PREPROCESSOR_OUTPUT_COMMENT 
+  |
   < ~[] >
 }
 

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -13,12 +13,13 @@ import java.util.Properties;
 import org.apache.commons.io.IOUtils;
 
 import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
-import net.sourceforge.pmd.lang.cpp.ast.Token;
 import net.sourceforge.pmd.util.IOUtil;
 
 /**
@@ -61,13 +62,14 @@ public class CPPTokenizer implements Tokenizer {
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(maybeSkipBlocks(buffer.toString()));
             reader = IOUtil.skipBOM(reader);
-            TokenManager tokenManager = languageVersionHandler
-                    .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
-                tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+            final TokenFilter tokenFilter = new JavaCCTokenFilter(
+                languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            
+            GenericToken currentToken = tokenFilter.getNextToken();
+            while (currentToken != null) {
+                tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
+                currentToken = tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java

@@ -5,6 +5,7 @@
 package net.sourceforge.pmd.cpd;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Properties;
@@ -19,7 +20,7 @@ public class CPPTokenizerTest {
     @Test
     public void testUTFwithBOM() {
         Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
-        assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
         assertEquals(15, tokens.size());
     }
 
@@ -29,9 +30,19 @@ public class CPPTokenizerTest {
                 + "int main()\n" + "{\n" + "    std::string text(\"ąęćśźńó\");\n" + "    std::cout << text;\n"
                 + "    return 0;\n" + "}\n";
         Tokens tokens = parse(code);
-        assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
         assertEquals(24, tokens.size());
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() {
+        String code = "#include <iostream>\n" + "#include <string>\n" + "\n" + "// CPD-OFF\n"
+                + "int main()\n" + "{\n" + "    std::string text(\"ąęćśźńó\");\n" + "    std::cout << text;\n"
+                + "    return 0;\n" + "// CPD-ON\n" + "}\n";
+        Tokens tokens = parse(code);
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
+        assertEquals(2, tokens.size()); // "}" + EOF
+    }
 
     @Test
     public void testMultiLineMacros() {