From b965090cc7a4c1253611bd25d717f211a61ed291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 15 Apr 2018 14:21:42 -0300 Subject: [PATCH] [plsql] Support comment-based CPD suppressions --- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 30 +++++++------------ .../pmd/cpd/PLSQLTokenizerTest.java | 21 +++++++++++-- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index 711b4305dd..74cfb32a41 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -9,19 +9,21 @@ import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; -import net.sourceforge.pmd.lang.ast.SimpleCharStream; +import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; +import net.sourceforge.pmd.cpd.token.TokenFilter; +import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager; import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants; -import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserTokenManager; import net.sourceforge.pmd.lang.plsql.ast.Token; public class PLSQLTokenizer implements Tokenizer { private static final Logger LOGGER = Logger.getLogger(PLSQLTokenizer.class.getName()); + // This is actually useless, the comments are special tokens, never taken into account by CPD + @Deprecated public static final String IGNORE_COMMENTS = "ignore_comments"; public static final String IGNORE_IDENTIFIERS = "ignore_identifiers"; public static final String IGNORE_LITERALS = "ignore_literals"; - private boolean ignoreComments; private boolean ignoreIdentifiers; private boolean ignoreLiterals; @@ -32,13 +34,13 @@ public class PLSQLTokenizer implements Tokenizer { * interested in comment variation, so we shall default ignoreComments * to true */ - ignoreComments = Boolean.parseBoolean(properties.getProperty(IGNORE_COMMENTS, "true")); ignoreIdentifiers = Boolean.parseBoolean(properties.getProperty(IGNORE_IDENTIFIERS, "false")); ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false")); } + @Deprecated public void setIgnoreComments(boolean ignore) { - this.ignoreComments = ignore; + // This is actually useless, the comments are special tokens, never taken into account by CPD } public void setIgnoreLiterals(boolean ignore) { @@ -65,7 +67,6 @@ public class PLSQLTokenizer implements Tokenizer { long addedTokens = 0; if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("PLSQLTokenizer: ignoreComments==" + ignoreComments); LOGGER.fine("PLSQLTokenizer: ignoreIdentifiers==" + ignoreIdentifiers); LOGGER.fine("PLSQLTokenizer: ignoreLiterals==" + ignoreLiterals); } @@ -73,21 +74,12 @@ public class PLSQLTokenizer implements Tokenizer { String fileName = sourceCode.getFileName(); StringBuilder sb = sourceCode.getCodeBuffer(); - PLSQLParserTokenManager tokenMgr = new PLSQLParserTokenManager( - new SimpleCharStream(new StringReader(sb.toString()))); - Token currentToken = tokenMgr.getNextToken(); - while (currentToken.image.length() > 0) { + TokenFilter tokenFilter = new JavaCCTokenFilter(new PLSQLTokenManager(new StringReader(sb.toString()))); + Token currentToken = (Token) tokenFilter.getNextToken(); + while (currentToken != null) { String image = currentToken.image; encounteredTokens++; - if (ignoreComments && (currentToken.kind == PLSQLParserConstants.SINGLE_LINE_COMMENT - || currentToken.kind == PLSQLParserConstants.MULTI_LINE_COMMENT - || currentToken.kind == PLSQLParserConstants.FORMAL_COMMENT - || currentToken.kind == PLSQLParserConstants.COMMENT - || currentToken.kind == PLSQLParserConstants.IN_MULTI_LINE_COMMENT - || currentToken.kind == PLSQLParserConstants.IN_FORMAL_COMMENT)) { - image = String.valueOf(currentToken.kind); - } if (ignoreIdentifiers && currentToken.kind == PLSQLParserConstants.IDENTIFIER) { image = String.valueOf(currentToken.kind); @@ -104,7 +96,7 @@ public class PLSQLTokenizer implements Tokenizer { tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine)); addedTokens++; - currentToken = tokenMgr.getNextToken(); + currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); if (LOGGER.isLoggable(Level.FINE)) { diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java index ace4b0517c..96d36dbd91 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java @@ -4,12 +4,15 @@ package net.sourceforge.pmd.cpd; +import static org.junit.Assert.assertEquals; + import java.io.IOException; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; +import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.testframework.AbstractTokenizerTest; public class PLSQLTokenizerTest extends AbstractTokenizerTest { @@ -33,8 +36,20 @@ public class PLSQLTokenizerTest extends AbstractTokenizerTest { this.expectedTokenCount = 1422; super.tokenizeTest(); } - - public static junit.framework.Test suite() { - return new junit.framework.JUnit4TestAdapter(PLSQLTokenizerTest.class); + + @Test + public void testIgnoreBetweenSpecialComments() throws IOException { + SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("-- CPD-OFF" + PMD.EOL + + "CREATE OR REPLACE" + PMD.EOL + + "PACKAGE \"test_schema\".\"BANK_DATA\"" + PMD.EOL + + "IS" + PMD.EOL + + "pi CONSTANT NUMBER := 3.1415;" + PMD.EOL + + "--CPD-ON" + PMD.EOL + + "END;" + )); + Tokens tokens = new Tokens(); + tokenizer.tokenize(sourceCode, tokens); + TokenEntry.getEOF(); + assertEquals(3, tokens.size()); // 3 tokens: "END" + ";" + EOF } }