diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index 27dab7e2e5..b976c90008 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -9,9 +9,10 @@ import java.io.StringReader; import org.apache.commons.io.IOUtils; +import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; +import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.objectivec.ObjectiveCLanguageModule; import net.sourceforge.pmd.lang.objectivec.ast.Token; @@ -29,13 +30,13 @@ public class ObjectiveCTokenizer implements Tokenizer { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); reader = new StringReader(buffer.toString()); - TokenManager tokenManager = languageVersionHandler + final TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) - .getTokenManager(sourceCode.getFileName(), reader); - Token currentToken = (Token) tokenManager.getNextToken(); - while (currentToken.image.length() > 0) { + .getTokenManager(sourceCode.getFileName(), reader)); + Token currentToken = (Token) tokenFilter.getNextToken(); + while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); - currentToken = (Token) tokenManager.getNextToken(); + currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); diff --git a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java index 7418a99b49..99ff788ba9 100644 --- a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java +++ b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java @@ -4,12 +4,15 @@ package net.sourceforge.pmd.cpd; +import static org.junit.Assert.assertEquals; + import java.io.IOException; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; +import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.testframework.AbstractTokenizerTest; //Tests if the ObjectiveC tokenizer supports identifiers with unicode characters @@ -34,4 +37,18 @@ public class UnicodeObjectiveCTokenizerTest extends AbstractTokenizerTest { this.expectedTokenCount = 10; super.tokenizeTest(); } + + @Test + public void testIgnoreBetweenSpecialComments() throws IOException { + SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader( + "// CPD-OFF" + PMD.EOL + + "static SecCertificateRef gNСServerLogonCertificate;" + PMD.EOL + + "// CPD-ON" + PMD.EOL + + "@end" + PMD.EOL + )); + Tokens tokens = new Tokens(); + tokenizer.tokenize(sourceCode, tokens); + TokenEntry.getEOF(); + assertEquals(2, tokens.size()); // 2 tokens: "@end" + EOF + } }