diff --git a/pmd-matlab/etc/grammar/matlab.jj b/pmd-matlab/etc/grammar/matlab.jj index 35dbc8625d..6f61dd0d42 100644 --- a/pmd-matlab/etc/grammar/matlab.jj +++ b/pmd-matlab/etc/grammar/matlab.jj @@ -39,27 +39,19 @@ PARSER_END(MatlabParser) "\r\n" : DEFAULT | "\n" : DEFAULT -| - "%{" : IN_COMMENT -| - "%" : IN_LINE_COMMENT - } - SKIP: -{ - "%}" : DEFAULT -} +MORE: +{ "%{": IN_COMMENT } - SKIP: -{ - "\n" : DEFAULT -} +SPECIAL_TOKEN: +{ } - MORE: -{ - < ~[] > -} + SPECIAL_TOKEN: +{ : DEFAULT } + + MORE: +{ < ~[] > } TOKEN : /* SEPARATORS AND OTHER USEFULL LANGUAGE CONSTRUCTS*/ { diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index b2157bd043..4b135d1be7 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -9,9 +9,10 @@ import java.io.StringReader; import org.apache.commons.io.IOUtils; +import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; +import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.matlab.MatlabLanguageModule; import net.sourceforge.pmd.lang.matlab.ast.Token; @@ -31,13 +32,13 @@ public class MatlabTokenizer implements Tokenizer { .getDefaultVersion().getLanguageVersionHandler(); reader = new StringReader(buffer.toString()); reader = IOUtil.skipBOM(reader); - TokenManager tokenManager = languageVersionHandler + final TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) - .getTokenManager(sourceCode.getFileName(), reader); - Token currentToken = (Token) tokenManager.getNextToken(); - while (currentToken.image.length() > 0) { + .getTokenManager(sourceCode.getFileName(), reader)); + Token currentToken = (Token) tokenFilter.getNextToken(); + while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); - currentToken = (Token) tokenManager.getNextToken(); + currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); diff --git a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java b/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java index e2ce74f064..0557063eb0 100644 --- a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java +++ b/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java @@ -4,12 +4,15 @@ package net.sourceforge.pmd.cpd; +import static org.junit.Assert.assertEquals; + import java.io.IOException; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; +import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.testframework.AbstractTokenizerTest; public class MatlabTokenizerTest extends AbstractTokenizerTest { @@ -33,4 +36,21 @@ public class MatlabTokenizerTest extends AbstractTokenizerTest { this.expectedTokenCount = 3925; super.tokenizeTest(); } + + @Test + public void testIgnoreBetweenSpecialComments() throws IOException { + SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("% CPD-OFF" + PMD.EOL + + "function g = vec(op, y)" + PMD.EOL + + " opy = op(y);" + PMD.EOL + + " if ( any(size(opy) > 1) )" + PMD.EOL + + " g = @loopWrapperArray;" + PMD.EOL + + " end" + PMD.EOL + + " % CPD-ON" + PMD.EOL + + "end" + )); + Tokens tokens = new Tokens(); + tokenizer.tokenize(sourceCode, tokens); + TokenEntry.getEOF(); + assertEquals(2, tokens.size()); // 2 tokens: "end" + EOF + } }