[javascript] Support comment based CPD suppressions

pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java

@@ -9,9 +9,10 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule;
 import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants;
@@ -30,14 +31,14 @@ public class EcmascriptTokenizer implements Tokenizer {
             LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME)
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(buffer.toString());
-            TokenManager tokenManager = languageVersionHandler
+            TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
                     .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
+                    .getTokenManager(sourceCode.getFileName(), reader));
-            Token currentToken = (Token) tokenManager.getNextToken();
+            Token currentToken = (Token) tokenFilter.getNextToken();
-            while (currentToken.image.length() > 0) {
+            while (currentToken != null) {
                 tokenEntries.add(
                         new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+                currentToken = (Token) tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());

pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java

@@ -33,6 +33,22 @@ public class EcmascriptTokenizerTest {
         t.tokenize(sourceCode, tokens);
         assertEquals(45, tokens.size());
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        final String code = "// CPD-OFF" + PMD.EOL
+            + "function switchToRealPassword() {" + PMD.EOL
+            + "var real = $('realPass');" + PMD.EOL
+            + " var prompt = $('promptPass');" + PMD.EOL
+            + "// CPD-ON" + PMD.EOL
+            + "}" + PMD.EOL;
+        
+        Tokenizer t = new EcmascriptTokenizer();
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(code));
+        Tokens tokens = new Tokens();
+        t.tokenize(sourceCode, tokens);
+        assertEquals(2, tokens.size()); // Only "}" and EOL
+    }
 
     /**
      * See: https://sourceforge.net/p/pmd/bugs/1239/