From 8364d97e92d5b953e669ba9402f341d6e3741a30 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 21 Dec 2018 11:26:05 +0100 Subject: [PATCH 01/90] [java] New Rule: UseTryWithResources Fixes #1405 --- docs/pages/release_notes.md | 7 + .../resources/category/java/bestpractices.xml | 56 +++++++ .../UseTryWithResourcesTest.java | 11 ++ .../bestpractices/xml/UseTryWithResources.xml | 144 ++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UseTryWithResourcesTest.java create mode 100644 pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index da0e17d3eb..59fa850112 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,12 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### New Rules + +* The new Java rule {% rule "java/bestpractices/UseTryWithResources" %) (`java-bestpractices`) searches + for try-blocks, that could be changed to a try-with-resources statement. This statement ensures that + each resource is closed at the end of the statement and is available since Java 7. + #### Modified Rules * The Java rule {% rule "java/codestyle/LocalVariableCouldBeFinal" %} (`java-codestyle`) has a new @@ -24,6 +30,7 @@ This is a {{ site.pmd.release_type }} release. * java-bestpractices * [#658](https://github.com/pmd/pmd/issues/658): \[java] OneDeclarationPerLine: False positive for loops + * [#1405](https://github.com/pmd/pmd/issues/1405): \[java] New Rule: UseTryWithResources - Replace close and IOUtils.closeQuietly with try-with-resources * java-codestyle * [#1513](https://github.com/pmd/pmd/issues/1513): \[java] LocalVariableCouldBeFinal: allow excluding the variable in a for-each loop * java-errorprone diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index 9e8ab6336f..5fc41be97d 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -1444,6 +1444,62 @@ public class Foo { + + +Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end +of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions +are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from +the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is +preserved. + + 3 + + + + + + + + + + + + + + + + + Code sample + 1 + 4 + + + + + With IOUtils.closeQuietly 1 + 1 + 4 + + + + + With IOUtils.closeQuietly 2 + 1 + 4 + + + + + Multiple Resources + 1 + 5 + + + + + Custom close methods + myClose2,myClose + 1 + 4 + + + From a8e136407262c2926426b6683fd92c16d5ee802e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 20 Jan 2019 09:46:51 +0100 Subject: [PATCH 02/90] [cpp] CPD: Lexical error in file (no file name provided) Fixes #1559 --- docs/pages/release_notes.md | 2 ++ .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 16 ++++----- .../sourceforge/pmd/cpd/CPPTokenizerTest.java | 36 ++++++++++++++++--- .../sourceforge/pmd/cpd/cpp/issue-1559.cpp | 11 ++++++ 4 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index fa0a5b63b4..8eb3a646dc 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -57,6 +57,8 @@ This is a {{ site.pmd.release_type }} release. * apex * [#1542](https://github.com/pmd/pmd/pull/1542): \[apex] Include the documentation category * [#1546](https://github.com/pmd/pmd/issues/1546): \[apex] PMD parsing exception for Apex classes using 'inherited sharing' keyword +* cpp + * [#1559](https://github.com/pmd/pmd/issues/1559): \[cpp] CPD: Lexical error in file (no file name provided) * java * [#1556](https://github.com/pmd/pmd/issues/1556): \[java] Default methods should not be considered abstract * [#1578](https://github.com/pmd/pmd/issues/1578): \[java] Private field is detected as public inside nested classes in interfaces diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 09ab82cd1a..26bdd74c40 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -14,7 +14,6 @@ import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.ast.GenericToken; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.cpp.CppTokenManager; import net.sourceforge.pmd.util.IOUtil; @@ -53,18 +52,19 @@ public class CPPTokenizer implements Tokenizer { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) { - final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader)); - + CppTokenManager tokenManager = new CppTokenManager(reader); + tokenManager.setFileName(sourceCode.getFileName()); + final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); + GenericToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine())); currentToken = tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode.getFileName()); - } catch (TokenMgrError | IOException err) { - err.printStackTrace(); - System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); + } catch (IOException e) { + e.printStackTrace(); + System.err.println("Error parsing " + sourceCode.getFileName()); + } finally { tokenEntries.add(TokenEntry.getEOF()); } } diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java index fb8fbd3360..15212eff7f 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java @@ -12,12 +12,18 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import org.apache.commons.io.IOUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import net.sourceforge.pmd.PMD; +import net.sourceforge.pmd.lang.ast.TokenMgrError; public class CPPTokenizerTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void testUTFwithBOM() { Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n"); @@ -76,15 +82,23 @@ public class CPPTokenizerTest { @Test public void testTokenizerWithSkipBlocksPattern() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); - Tokens tokens = parse(test, true, "#if debug|#endif"); - assertEquals(31, tokens.size()); + try { + Tokens tokens = parse(test, true, "#if debug|#endif"); + assertEquals(31, tokens.size()); + } catch (TokenMgrError ignored) { + // ignored + } } @Test public void testTokenizerWithoutSkipBlocks() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); - Tokens tokens = parse(test, false); - assertEquals(37, tokens.size()); + try { + Tokens tokens = parse(test, false); + assertEquals(37, tokens.size()); + } catch (TokenMgrError ignored) { + // ignored + } } @Test @@ -128,6 +142,20 @@ public class CPPTokenizerTest { assertEquals(9, tokens.size()); } + @Test + public void testLexicalErrorFilename() throws Exception { + Properties properties = new Properties(); + properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false)); + String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8); + SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp")); + CPPTokenizer tokenizer = new CPPTokenizer(); + tokenizer.setProperties(properties); + + expectedException.expect(TokenMgrError.class); + expectedException.expectMessage("Lexical error in file issue-1559.cpp at"); + tokenizer.tokenize(code, new Tokens()); + } + private Tokens parse(String snippet) { return parse(snippet, false); } diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp new file mode 100644 index 0000000000..010ec09fc6 --- /dev/null +++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp @@ -0,0 +1,11 @@ +namespace ABC +{ + namespace DEF + { + +#ifdef USE_QT + const char* perPixelQml = R"QML( +)QML"; + } +} +#endif // USE_QT From b745f331b8ca490e0046b566b5a94a2eb10aca84 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 20 Jan 2019 09:59:59 +0100 Subject: [PATCH 03/90] CPD: Fix error handling for lexical errors * TokenMgrError must not be caught by the tokenizer. This is handled by CPD itself * The token managers need to know the filename for proper error messages --- .../pmd/cpd/EcmascriptTokenizer.java | 9 ++------- .../sourceforge/pmd/cpd/MatlabTokenizer.java | 11 +++++----- .../pmd/cpd/ObjectiveCTokenizer.java | 13 +++++------- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 17 +++++++++++++--- .../sourceforge/pmd/cpd/PythonTokenizer.java | 20 +++++++------------ 5 files changed, 33 insertions(+), 37 deletions(-) diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index 3c236c57e4..bafbde5a9f 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -12,7 +12,6 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants; import net.sourceforge.pmd.lang.ecmascript5.ast.Token; @@ -37,14 +36,10 @@ public class EcmascriptTokenizer implements Tokenizer { new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode.getFileName()); - } catch (TokenMgrError err) { - err.printStackTrace(); - System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); - tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); + } finally { + tokenEntries.add(TokenEntry.getEOF()); } } diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index b0eb103f4e..7879052b99 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -10,7 +10,6 @@ import java.io.StringReader; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.matlab.MatlabTokenManager; import net.sourceforge.pmd.lang.matlab.ast.Token; import net.sourceforge.pmd.util.IOUtil; @@ -24,17 +23,17 @@ public class MatlabTokenizer implements Tokenizer { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { - final TokenFilter tokenFilter = new JavaCCTokenFilter(new MatlabTokenManager(reader)); + MatlabTokenManager tokenManager = new MatlabTokenManager(reader); + tokenManager.setFileName(sourceCode.getFileName()); + final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode.getFileName()); - } catch (TokenMgrError | IOException err) { + } catch (IOException err) { err.printStackTrace(); - System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); + } finally { tokenEntries.add(TokenEntry.getEOF()); } } diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index 2bb7e7deac..d224ef211f 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -10,7 +10,6 @@ import java.io.StringReader; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.objectivec.ObjectiveCTokenManager; import net.sourceforge.pmd.lang.objectivec.ast.Token; @@ -23,20 +22,18 @@ public class ObjectiveCTokenizer implements Tokenizer { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = new StringReader(buffer.toString())) { - final TokenFilter tokenFilter = new JavaCCTokenFilter(new ObjectiveCTokenManager(reader)); + ObjectiveCTokenManager tokenManager = new ObjectiveCTokenManager(reader); + tokenManager.setFileName(sourceCode.getFileName()); + final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode.getFileName()); - } catch (TokenMgrError err) { - err.printStackTrace(); - System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); - tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); + } finally { + tokenEntries.add(TokenEntry.getEOF()); } } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index 74cfb32a41..ab090acac1 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -11,7 +11,10 @@ import java.util.logging.Logger; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager; +import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.LanguageVersionHandler; +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.plsql.PLSQLLanguageModule; import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants; import net.sourceforge.pmd.lang.plsql.ast.Token; @@ -72,9 +75,8 @@ public class PLSQLTokenizer implements Tokenizer { } String fileName = sourceCode.getFileName(); - StringBuilder sb = sourceCode.getCodeBuffer(); - TokenFilter tokenFilter = new JavaCCTokenFilter(new PLSQLTokenManager(new StringReader(sb.toString()))); + TokenFilter tokenFilter = createTokenFilter(sourceCode); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { String image = currentToken.image; @@ -105,4 +107,13 @@ public class PLSQLTokenizer implements Tokenizer { } } + private JavaCCTokenFilter createTokenFilter(final SourceCode sourceCode) { + final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); + final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PLSQLLanguageModule.NAME) + .getDefaultVersion().getLanguageVersionHandler(); + final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) + .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); + return new JavaCCTokenFilter(tokenMgr); + } + } diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 9996d61985..6b518c2061 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -10,10 +10,7 @@ import java.io.StringReader; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.LanguageRegistry; -import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.python.PythonLanguageModule; +import net.sourceforge.pmd.lang.python.PythonTokenManager; import net.sourceforge.pmd.lang.python.ast.Token; import net.sourceforge.pmd.util.IOUtil; @@ -26,21 +23,18 @@ public class PythonTokenizer implements Tokenizer { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { - LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME) - .getDefaultVersion().getLanguageVersionHandler(); - TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler - .getParser(languageVersionHandler.getDefaultParserOptions()) - .getTokenManager(sourceCode.getFileName(), reader)); + PythonTokenManager tokenManager = new PythonTokenManager(reader); + tokenManager.setFileName(sourceCode.getFileName()); + + TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } - tokenEntries.add(TokenEntry.getEOF()); - System.err.println("Added " + sourceCode); - } catch (TokenMgrError | IOException err) { + } catch (IOException err) { err.printStackTrace(); - System.err.println("Skipping " + sourceCode + " due to parse error"); + } finally { tokenEntries.add(TokenEntry.getEOF()); } } From a79d4aeb4f5ec5b417f1f069ce55e20deb4c6a39 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 20 Jan 2019 11:47:59 +0100 Subject: [PATCH 04/90] Fix asserts --- .../sourceforge/pmd/cpd/CPPTokenizerTest.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java index 15212eff7f..75f45768c6 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java @@ -75,30 +75,32 @@ public class CPPTokenizerTest { @Test public void testTokenizerWithSkipBlocks() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); - Tokens tokens = parse(test, true); + Tokens tokens = parse(test, true, new Tokens()); assertEquals(19, tokens.size()); } @Test public void testTokenizerWithSkipBlocksPattern() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); + Tokens tokens = new Tokens(); try { - Tokens tokens = parse(test, true, "#if debug|#endif"); - assertEquals(31, tokens.size()); + parse(test, true, "#if debug|#endif", tokens); } catch (TokenMgrError ignored) { // ignored } + assertEquals(31, tokens.size()); } @Test public void testTokenizerWithoutSkipBlocks() throws Exception { String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8); + Tokens tokens = new Tokens(); try { - Tokens tokens = parse(test, false); - assertEquals(37, tokens.size()); + parse(test, false, tokens); } catch (TokenMgrError ignored) { // ignored } + assertEquals(37, tokens.size()); } @Test @@ -157,14 +159,14 @@ public class CPPTokenizerTest { } private Tokens parse(String snippet) { - return parse(snippet, false); + return parse(snippet, false, new Tokens()); } - private Tokens parse(String snippet, boolean skipBlocks) { - return parse(snippet, skipBlocks, null); + private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) { + return parse(snippet, skipBlocks, null, tokens); } - private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) { + private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) { Properties properties = new Properties(); properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks)); if (skipPattern != null) { @@ -175,7 +177,6 @@ public class CPPTokenizerTest { tokenizer.setProperties(properties); SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet)); - Tokens tokens = new Tokens(); tokenizer.tokenize(code, tokens); return tokens; } From 17fe6694ba4b4dfba9eeff8ff56a7836b144b069 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 23 Jan 2019 20:09:13 +0100 Subject: [PATCH 05/90] [core] Failed build on Windows with source zip archive Fixes #1462 --- docs/pages/release_notes.md | 1 + .../pmd/renderers/YAHTMLRendererTest.java | 8 +++++++- .../net/sourceforge/pmd/docs/MockedFileWriter.java | 9 +++++++++ .../sourceforge/pmd/docs/RuleDocGeneratorTest.java | 14 ++++++++------ .../sourceforge/pmd/docs/SidebarGeneratorTest.java | 4 +++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 8a2bae25df..28becad665 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -56,6 +56,7 @@ This is a {{ site.pmd.release_type }} release. * all * [#1196](https://github.com/pmd/pmd/issues/1196): \[core] CPD results not consistent between runs + * [#1462](https://github.com/pmd/pmd/issues/1462): \[core] Failed build on Windows with source zip archive * apex * [#1542](https://github.com/pmd/pmd/pull/1542): \[apex] Include the documentation category * [#1546](https://github.com/pmd/pmd/issues/1546): \[apex] PMD parsing exception for Apex classes using 'inherited sharing' keyword diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/YAHTMLRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/YAHTMLRendererTest.java index 7e968a7c30..4667b538ee 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/YAHTMLRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/YAHTMLRendererTest.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.junit.After; @@ -104,13 +105,18 @@ public class YAHTMLRendererTest extends AbstractRendererTst { try (FileInputStream in = new FileInputStream(new File(outputDir, file)); InputStream expectedIn = YAHTMLRendererTest.class.getResourceAsStream("yahtml/" + file)) { String data = IOUtils.toString(in, StandardCharsets.UTF_8); - String expected = IOUtils.toString(expectedIn, StandardCharsets.UTF_8); + String expected = normalizeLineSeparators(IOUtils.toString(expectedIn, StandardCharsets.UTF_8)); assertEquals("File " + file + " is different", expected, data); } } } + private static String normalizeLineSeparators(String s) { + return s.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_WINDOWS), IOUtils.LINE_SEPARATOR_UNIX) + .replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_UNIX), PMD.EOL); + } + @Override public Renderer getRenderer() { Renderer result = new YAHTMLRenderer(); diff --git a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/MockedFileWriter.java b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/MockedFileWriter.java index a533e136f2..7ff3dd6a47 100644 --- a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/MockedFileWriter.java +++ b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/MockedFileWriter.java @@ -8,9 +8,13 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import net.sourceforge.pmd.PMD; + public class MockedFileWriter implements FileWriter { public static class FileEntry { @@ -43,4 +47,9 @@ public class MockedFileWriter implements FileWriter { public void reset() { data.clear(); } + + public static String normalizeLineSeparators(String s) { + return s.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_WINDOWS), IOUtils.LINE_SEPARATOR_UNIX) + .replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_UNIX), PMD.EOL); + } } diff --git a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleDocGeneratorTest.java b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleDocGeneratorTest.java index 2028845677..65ee4ef4a7 100644 --- a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleDocGeneratorTest.java +++ b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleDocGeneratorTest.java @@ -67,6 +67,11 @@ public class RuleDocGeneratorTest { }); } + private static String loadResource(String name) throws IOException { + return MockedFileWriter.normalizeLineSeparators( + IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream(name), StandardCharsets.UTF_8)); + } + @Test public void testSingleRuleset() throws RuleSetNotFoundException, IOException { RuleDocGenerator generator = new RuleDocGenerator(writer, root); @@ -82,17 +87,14 @@ public class RuleDocGeneratorTest { assertEquals(3, writer.getData().size()); FileEntry languageIndex = writer.getData().get(0); assertTrue(FilenameUtils.normalize(languageIndex.getFilename(), true).endsWith("docs/pages/pmd/rules/java.md")); - assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/java.md"), StandardCharsets.UTF_8), - languageIndex.getContent()); + assertEquals(loadResource("/expected/java.md"), languageIndex.getContent()); FileEntry ruleSetIndex = writer.getData().get(1); assertTrue(FilenameUtils.normalize(ruleSetIndex.getFilename(), true).endsWith("docs/pages/pmd/rules/java/sample.md")); - assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/sample.md"), StandardCharsets.UTF_8), - ruleSetIndex.getContent()); + assertEquals(loadResource("/expected/sample.md"), ruleSetIndex.getContent()); FileEntry sidebar = writer.getData().get(2); assertTrue(FilenameUtils.normalize(sidebar.getFilename(), true).endsWith("docs/_data/sidebars/pmd_sidebar.yml")); - assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/pmd_sidebar.yml"), StandardCharsets.UTF_8), - sidebar.getContent()); + assertEquals(loadResource("/expected/pmd_sidebar.yml"), sidebar.getContent()); } } diff --git a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/SidebarGeneratorTest.java b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/SidebarGeneratorTest.java index b3d96b4e19..474e8683f6 100644 --- a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/SidebarGeneratorTest.java +++ b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/SidebarGeneratorTest.java @@ -55,6 +55,8 @@ public class SidebarGeneratorTest { } String yaml = new Yaml(options).dump(result); - assertEquals(IOUtils.toString(SidebarGeneratorTest.class.getResourceAsStream("sidebar.yml"), StandardCharsets.UTF_8), yaml); + String expected = MockedFileWriter.normalizeLineSeparators( + IOUtils.toString(SidebarGeneratorTest.class.getResourceAsStream("sidebar.yml"), StandardCharsets.UTF_8)); + assertEquals(expected, yaml); } } From 08b7dd49dda043f3ac0078c8e071dc7e86e27ada Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 26 Jan 2019 11:24:32 +0100 Subject: [PATCH 06/90] [core] Add a generic JavaCCTokenizer for CPD --- .../sourceforge/pmd/cpd/AntlrTokenizer.java | 1 + .../sourceforge/pmd/cpd/JavaCCTokenizer.java | 42 +++++++++ .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 38 +++----- .../pmd/cpd/CPPTokenizerContinuationTest.java | 3 +- .../sourceforge/pmd/cpd/CPPTokenizerTest.java | 11 ++- .../sourceforge/pmd/cpd/JavaTokenizer.java | 66 +++++++------- .../pmd/cpd/JavaTokensTokenizerTest.java | 2 +- .../pmd/cpd/MatchAlgorithmTest.java | 5 +- .../pmd/cpd/EcmascriptTokenizer.java | 49 ++++------- .../sourceforge/pmd/cpd/MatlabTokenizer.java | 25 +----- .../pmd/cpd/ObjectiveCTokenizer.java | 26 ++---- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 87 +++++-------------- .../sourceforge/pmd/cpd/PythonTokenizer.java | 26 +----- 13 files changed, 154 insertions(+), 227 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index aad0d2379a..d1655da3a1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -23,6 +23,7 @@ public abstract class AntlrTokenizer implements Tokenizer { public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) { AntlrTokenManager tokenManager = getLexerForSource(sourceCode); + tokenManager.setFileName(sourceCode.getFileName()); try { AntlrToken token = (AntlrToken) tokenManager.getNextToken(); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java new file mode 100644 index 0000000000..e83f48cae4 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java @@ -0,0 +1,42 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import java.io.IOException; + +import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; +import net.sourceforge.pmd.cpd.token.TokenFilter; +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.GenericToken; + +public abstract class JavaCCTokenizer implements Tokenizer { + + protected abstract TokenManager getLexerForSource(SourceCode sourceCode); + + protected TokenFilter getTokenFilter(TokenManager tokenManager) { + return new JavaCCTokenFilter(tokenManager); + } + + protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) { + return new TokenEntry(currentToken.getImage(), filename, currentToken.getBeginLine()); + } + + @Override + public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException { + TokenManager tokenManager = getLexerForSource(sourceCode); + tokenManager.setFileName(sourceCode.getFileName()); + try { + final TokenFilter tokenFilter = getTokenFilter(tokenManager); + + GenericToken currentToken = tokenFilter.getNextToken(); + while (currentToken != null) { + tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName())); + currentToken = tokenFilter.getNextToken(); + } + } finally { + tokenEntries.add(TokenEntry.getEOF()); + } + } +} diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 26bdd74c40..18b2c93813 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -6,21 +6,18 @@ package net.sourceforge.pmd.cpd; import java.io.BufferedReader; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.util.Properties; import net.sourceforge.pmd.PMD; -import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; -import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.cpp.CppTokenManager; import net.sourceforge.pmd.util.IOUtil; /** * The C++ tokenizer. */ -public class CPPTokenizer implements Tokenizer { +public class CPPTokenizer extends JavaCCTokenizer { private boolean skipBlocks = true; private String skipBlocksStart; @@ -48,27 +45,6 @@ public class CPPTokenizer implements Tokenizer { } } - @Override - public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { - StringBuilder buffer = sourceCode.getCodeBuffer(); - try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) { - CppTokenManager tokenManager = new CppTokenManager(reader); - tokenManager.setFileName(sourceCode.getFileName()); - final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager); - - GenericToken currentToken = tokenFilter.getNextToken(); - while (currentToken != null) { - tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine())); - currentToken = tokenFilter.getNextToken(); - } - } catch (IOException e) { - e.printStackTrace(); - System.err.println("Error parsing " + sourceCode.getFileName()); - } finally { - tokenEntries.add(TokenEntry.getEOF()); - } - } - private String maybeSkipBlocks(String test) throws IOException { if (!skipBlocks) { return test; @@ -92,4 +68,14 @@ public class CPPTokenizer implements Tokenizer { } return filtered.toString(); } + + @Override + protected TokenManager getLexerForSource(SourceCode sourceCode) { + try { + StringBuilder buffer = sourceCode.getCodeBuffer(); + return new CppTokenManager(IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java index f5cd31bd1b..188609febd 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import java.io.IOException; import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -117,7 +118,7 @@ public class CPPTokenizerContinuationTest { .getResourceAsStream("cpp/" + name), StandardCharsets.UTF_8); } - private Tokens parse(String code) { + private Tokens parse(String code) throws IOException { CPPTokenizer tokenizer = new CPPTokenizer(); tokenizer.setProperties(new Properties()); Tokens tokens = new Tokens(); diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java index 75f45768c6..4bffb5208d 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java @@ -8,6 +8,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Properties; @@ -159,14 +160,18 @@ public class CPPTokenizerTest { } private Tokens parse(String snippet) { - return parse(snippet, false, new Tokens()); + try { + return parse(snippet, false, new Tokens()); + } catch (IOException e) { + throw new RuntimeException(e); + } } - private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) { + private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) throws IOException { return parse(snippet, skipBlocks, null, tokens); } - private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) { + private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) throws IOException { Properties properties = new Properties(); properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks)); if (skipPattern != null) { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index d758704ba3..9f3ea6d792 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -4,21 +4,21 @@ package net.sourceforge.pmd.cpd; +import java.io.IOException; import java.io.StringReader; import java.util.Deque; import java.util.LinkedList; import java.util.Properties; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; -import net.sourceforge.pmd.lang.LanguageRegistry; -import net.sourceforge.pmd.lang.LanguageVersionHandler; +import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; -import net.sourceforge.pmd.lang.java.JavaLanguageModule; +import net.sourceforge.pmd.lang.java.JavaTokenManager; import net.sourceforge.pmd.lang.java.ast.JavaParserConstants; import net.sourceforge.pmd.lang.java.ast.Token; -public class JavaTokenizer implements Tokenizer { +public class JavaTokenizer extends JavaCCTokenizer { public static final String CPD_START = "\"CPD-START\""; public static final String CPD_END = "\"CPD-END\""; @@ -27,6 +27,8 @@ public class JavaTokenizer implements Tokenizer { private boolean ignoreLiterals; private boolean ignoreIdentifiers; + private ConstructorDetector constructorDetector; + public void setProperties(Properties properties) { ignoreAnnotations = Boolean.parseBoolean(properties.getProperty(IGNORE_ANNOTATIONS, "false")); ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false")); @@ -34,48 +36,42 @@ public class JavaTokenizer implements Tokenizer { } @Override - public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { - final String fileName = sourceCode.getFileName(); - final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode); - final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers); - - Token currentToken = (Token) tokenFilter.getNextToken(); - while (currentToken != null) { - processToken(tokenEntries, fileName, currentToken, constructorDetector); - currentToken = (Token) tokenFilter.getNextToken(); - } - tokenEntries.add(TokenEntry.getEOF()); + public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException { + constructorDetector = new ConstructorDetector(ignoreIdentifiers); + super.tokenize(sourceCode, tokenEntries); } - private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { + @Override + protected TokenManager getLexerForSource(SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); - // Note that Java version is irrelevant for tokenizing - final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) - .getVersion("1.4").getLanguageVersionHandler(); - final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) - .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); - return new JavaTokenFilter(tokenMgr, ignoreAnnotations); + return new JavaTokenManager(new StringReader(stringBuilder.toString())); } - private void processToken(Tokens tokenEntries, String fileName, Token currentToken, - ConstructorDetector constructorDetector) { - String image = currentToken.image; + @Override + protected TokenFilter getTokenFilter(TokenManager tokenManager) { + return new JavaTokenFilter(tokenManager, ignoreAnnotations); + } - constructorDetector.restoreConstructorToken(tokenEntries, currentToken); + @Override + protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String fileName) { + String image = currentToken.getImage(); + Token javaToken = (Token) currentToken; - if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL - || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL - || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL - || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) { - image = String.valueOf(currentToken.kind); + constructorDetector.restoreConstructorToken(tokenEntries, javaToken); + + if (ignoreLiterals && (javaToken.kind == JavaParserConstants.STRING_LITERAL + || javaToken.kind == JavaParserConstants.CHARACTER_LITERAL + || javaToken.kind == JavaParserConstants.DECIMAL_LITERAL + || javaToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) { + image = String.valueOf(javaToken.kind); } - if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) { - image = String.valueOf(currentToken.kind); + if (ignoreIdentifiers && javaToken.kind == JavaParserConstants.IDENTIFIER) { + image = String.valueOf(javaToken.kind); } - constructorDetector.processToken(currentToken); + constructorDetector.processToken(javaToken); - tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine)); + return new TokenEntry(image, fileName, currentToken.getBeginLine()); } public void setIgnoreLiterals(boolean ignore) { diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java index 3291c53ad6..74cb844c65 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java @@ -88,7 +88,7 @@ public class JavaTokensTokenizerTest { } @Test - public void testIgnoreComments() { + public void testIgnoreComments() throws IOException { JavaTokenizer t = new JavaTokenizer(); t.setIgnoreAnnotations(false); SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java index 972b90d1f7..6c2e4937bb 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -32,7 +33,7 @@ public class MatchAlgorithmTest { } @Test - public void testSimple() { + public void testSimple() throws IOException { JavaTokenizer tokenizer = new JavaTokenizer(); SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(getSampleCode(), "Foo.java")); Tokens tokens = new Tokens(); @@ -63,7 +64,7 @@ public class MatchAlgorithmTest { } @Test - public void testIgnore() { + public void testIgnore() throws IOException { JavaTokenizer tokenizer = new JavaTokenizer(); tokenizer.setIgnoreLiterals(true); tokenizer.setIgnoreIdentifiers(true); diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index bafbde5a9f..fed848801b 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -4,51 +4,38 @@ package net.sourceforge.pmd.cpd; -import java.io.IOException; -import java.io.Reader; import java.io.StringReader; -import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; -import net.sourceforge.pmd.cpd.token.TokenFilter; -import net.sourceforge.pmd.lang.LanguageRegistry; -import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule; +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.ecmascript5.Ecmascript5TokenManager; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants; import net.sourceforge.pmd.lang.ecmascript5.ast.Token; +import net.sourceforge.pmd.util.IOUtil; /** * The Ecmascript Tokenizer */ -public class EcmascriptTokenizer implements Tokenizer { +public class EcmascriptTokenizer extends JavaCCTokenizer { @Override - public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { + protected TokenManager getLexerForSource(SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); - try (Reader reader = new StringReader(buffer.toString())) { - LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME) - .getDefaultVersion().getLanguageVersionHandler(); - TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler - .getParser(languageVersionHandler.getDefaultParserOptions()) - .getTokenManager(sourceCode.getFileName(), reader)); - Token currentToken = (Token) tokenFilter.getNextToken(); - while (currentToken != null) { - tokenEntries.add( - new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine)); - currentToken = (Token) tokenFilter.getNextToken(); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - tokenEntries.add(TokenEntry.getEOF()); - } + return new Ecmascript5TokenManager(IOUtil.skipBOM(new StringReader(buffer.toString()))); } - private String getTokenImage(Token token) { + @Override + protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) { + return new TokenEntry(getTokenImage(currentToken), filename, currentToken.getBeginLine()); + } + + private String getTokenImage(GenericToken token) { + Token jsToken = (Token) token; // Remove line continuation characters from string literals - if (token.kind == Ecmascript5ParserConstants.STRING_LITERAL - || token.kind == Ecmascript5ParserConstants.UNTERMINATED_STRING_LITERAL) { - return token.image.replaceAll("(? Date: Sat, 26 Jan 2019 11:59:04 +0100 Subject: [PATCH 07/90] [java] UseTryWithResources: Fix false negative --- .../resources/category/java/bestpractices.xml | 2 +- .../bestpractices/xml/UseTryWithResources.xml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index 5fc41be97d..2e9a970fb6 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -1465,7 +1465,7 @@ preserved. diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml index 691477e1b1..70942a71db 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml @@ -141,4 +141,21 @@ public class TryWithResources { } ]]> + + + False negative with two resources + 1 + 4 + + From ac1ea34c4c21283a567499f9b86630097783059b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 26 Jan 2019 12:16:19 +0100 Subject: [PATCH 08/90] [java] UseTryWithResources: fix false positive with no autocloseable --- .../resources/category/java/bestpractices.xml | 5 +++ .../bestpractices/xml/UseTryWithResources.xml | 43 ++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index 2e9a970fb6..f4b6f525af 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -1448,6 +1448,7 @@ public class Foo { language="java" minimumLanguageVersion="1.7" since="6.11.0" + typeResolution="true" message="Consider use the try-with-resources statement instead of explicitly closing the resource" class="net.sourceforge.pmd.lang.rule.XPathRule" externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_bestpractices.html#usetrywithresources"> @@ -1467,6 +1468,10 @@ preserved. diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml index 70942a71db..5761d11723 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml @@ -6,8 +6,10 @@ Code sample 1 - 4 + 6 With IOUtils.closeQuietly 1 1 - 4 + 6 With IOUtils.closeQuietly 2 1 - 4 + 6 Multiple Resources 1 - 5 + 7 Custom close methods myClose2,myClose 1 - 4 + 6 False negative with two resources 1 - 4 + 6 + + + + False positive with no autocloseable + 0 + From 2465e5395c2ed50363c0f16e0359b1f84d30425c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 26 Jan 2019 13:02:40 +0100 Subject: [PATCH 09/90] Update pmd-java/src/main/resources/category/java/bestpractices.xml Co-Authored-By: adangel --- pmd-java/src/main/resources/category/java/bestpractices.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index 61f42b8651..03285b22f6 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -1524,7 +1524,7 @@ public class Foo { minimumLanguageVersion="1.7" since="6.11.0" typeResolution="true" - message="Consider use the try-with-resources statement instead of explicitly closing the resource" + message="Consider using a try-with-resources statement instead of explicitly closing the resource" class="net.sourceforge.pmd.lang.rule.XPathRule" externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_bestpractices.html#usetrywithresources"> From 89c8f47b2b2b1cfceba7e73f0b77118426f55f0c Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 26 Jan 2019 19:19:46 +0100 Subject: [PATCH 10/90] Move JavaCCTokenizer to internal package --- .../sourceforge/pmd/cpd/{ => internal}/JavaCCTokenizer.java | 6 +++++- .../src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java | 1 + .../main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java | 1 + .../java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java | 1 + .../main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java | 1 + .../java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java | 1 + .../main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java | 1 + .../main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java | 1 + 8 files changed, 12 insertions(+), 1 deletion(-) rename pmd-core/src/main/java/net/sourceforge/pmd/cpd/{ => internal}/JavaCCTokenizer.java (87%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java similarity index 87% rename from pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java rename to pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index e83f48cae4..03adab05c5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -2,10 +2,14 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.cpd.internal; import java.io.IOException; +import net.sourceforge.pmd.cpd.SourceCode; +import net.sourceforge.pmd.cpd.TokenEntry; +import net.sourceforge.pmd.cpd.Tokenizer; +import net.sourceforge.pmd.cpd.Tokens; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 18b2c93813..5573c736df 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -10,6 +10,7 @@ import java.io.StringReader; import java.util.Properties; import net.sourceforge.pmd.PMD; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.cpp.CppTokenManager; import net.sourceforge.pmd.util.IOUtil; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 9f3ea6d792..cab722447b 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -10,6 +10,7 @@ import java.util.Deque; import java.util.LinkedList; import java.util.Properties; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index fed848801b..64c7914912 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd; import java.io.StringReader; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ecmascript5.Ecmascript5TokenManager; diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index 8dc00b8909..26299edb6c 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd; import java.io.StringReader; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.matlab.MatlabTokenManager; import net.sourceforge.pmd.util.IOUtil; diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index fd96c809b5..32ea9c4460 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd; import java.io.StringReader; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.objectivec.ObjectiveCTokenManager; import net.sourceforge.pmd.util.IOUtil; diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index c605e276a7..b44122309b 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd; import java.io.StringReader; import java.util.Properties; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager; diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 8eb75f7277..e88b1476cf 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd; import java.io.StringReader; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.python.PythonTokenManager; import net.sourceforge.pmd.util.IOUtil; From ea022f0e68f629ad48e3b9747aa6be56f0e9d35c Mon Sep 17 00:00:00 2001 From: Bobby Wertman Date: Sun, 27 Jan 2019 10:55:45 -0500 Subject: [PATCH 11/90] [java] Fix lombok.AllArgsConstructor support --- .../codestyle/AtLeastOneConstructorRule.java | 2 +- .../codestyle/xml/AtLeastOneConstructor.xml | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/AtLeastOneConstructorRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/AtLeastOneConstructorRule.java index 2983c4eaad..ee06ee9951 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/AtLeastOneConstructorRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/AtLeastOneConstructorRule.java @@ -33,7 +33,7 @@ public class AtLeastOneConstructorRule extends AbstractIgnoredAnnotationRule { "lombok.Builder", "lombok.NoArgsConstructor", "lombok.RequiredArgsConstructor", - "lombok.AllArgsConstructorAtLeastOneConstructor"); + "lombok.AllArgsConstructor"); } @Override diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/AtLeastOneConstructor.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/AtLeastOneConstructor.xml index fe3fab3c72..ea0e7b2a4c 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/AtLeastOneConstructor.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/AtLeastOneConstructor.xml @@ -117,13 +117,57 @@ public class TestAtLeastOneConstructor { ]]> - Ignore classes with lombok-generated constructors + Ignore classes with lombok-generated constructors (Value) 0 + + + Ignore classes with lombok-generated constructors (Required) + 0 + + + + Ignore classes with lombok-generated constructors (No) + 0 + + + + Ignore classes with lombok-generated constructors (All) + 0 + + + + Ignore classes with lombok-generated constructors (Builder) + 0 + From 8a46cea437fab4b04889f58635454eb850b420ab Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Mon, 28 Jan 2019 01:26:10 -0300 Subject: [PATCH 12/90] Added AntlrTokenFilter --- .../sourceforge/pmd/cpd/AntlrTokenizer.java | 34 ++++++++++-------- .../pmd/cpd/token/AntlrHiddenTokenFilter.java | 16 +++++++++ .../pmd/cpd/token/AntlrTokenFilter.java | 17 +++++++++ .../pmd/cpd/GoCPDTokenizerTest.java | 36 +++++++++++++++++++ .../net/sourceforge/pmd/cpd/hello.go | 29 +++++++++++++++ 5 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java create mode 100644 pmd-go/src/test/java/net/sourceforge/pmd/cpd/GoCPDTokenizerTest.java create mode 100644 pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index aad0d2379a..93fabf6049 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -4,13 +4,15 @@ package net.sourceforge.pmd.cpd; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.Token; - +import net.sourceforge.pmd.cpd.token.AntlrHiddenTokenFilter; import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; + +import static org.antlr.v4.runtime.Token.EOF; /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. @@ -23,18 +25,13 @@ public abstract class AntlrTokenizer implements Tokenizer { public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) { AntlrTokenManager tokenManager = getLexerForSource(sourceCode); + AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager); try { - AntlrToken token = (AntlrToken) tokenManager.getNextToken(); - - while (token.getType() != Token.EOF) { - if (!token.isHidden()) { - final TokenEntry tokenEntry = - new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine()); - - tokenEntries.add(tokenEntry); - } - token = (AntlrToken) tokenManager.getNextToken(); + AntlrToken currentToken = tokenFilter.getNextToken(); + while (currentToken != null && currentToken.getType() != EOF) { + processToken(tokenEntries, tokenManager.getFileName(), currentToken); + currentToken = tokenFilter.getNextToken(); } } catch (final AntlrTokenManager.ANTLRSyntaxError err) { // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled @@ -47,8 +44,17 @@ public abstract class AntlrTokenizer implements Tokenizer { } } + protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + return new AntlrHiddenTokenFilter(tokenManager); + } + /* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); return CharStreams.fromString(buffer.toString()); } + + private void processToken(Tokens tokenEntries, String fileName, AntlrToken token) { + final TokenEntry tokenEntry = new TokenEntry(token.getImage(), fileName, token.getBeginLine()); + tokenEntries.add(tokenEntry); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java new file mode 100644 index 0000000000..27acac6d34 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java @@ -0,0 +1,16 @@ +package net.sourceforge.pmd.cpd.token; + +import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; + +public class AntlrHiddenTokenFilter extends AntlrTokenFilter { + + public AntlrHiddenTokenFilter(final AntlrTokenManager tokenManager) { + super(tokenManager); + } + + @Override + public AntlrToken getNextToken() { + AntlrToken token = super.getNextToken(); + return token.isHidden() ? getNextToken() : token; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java new file mode 100644 index 0000000000..74ae610bef --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -0,0 +1,17 @@ +package net.sourceforge.pmd.cpd.token; + +import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; + +public class AntlrTokenFilter implements TokenFilter { + + private TokenFilter tokenFilter; + + public AntlrTokenFilter(final AntlrTokenManager tokenManager) { + this.tokenFilter = new JavaCCTokenFilter(tokenManager); + } + + @Override + public AntlrToken getNextToken() { + return (AntlrToken) tokenFilter.getNextToken(); + } +} diff --git a/pmd-go/src/test/java/net/sourceforge/pmd/cpd/GoCPDTokenizerTest.java b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/GoCPDTokenizerTest.java new file mode 100644 index 0000000000..e46866d012 --- /dev/null +++ b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/GoCPDTokenizerTest.java @@ -0,0 +1,36 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; + +import net.sourceforge.pmd.testframework.AbstractTokenizerTest; + +public class GoCPDTokenizerTest extends AbstractTokenizerTest { + + private static final String FILENAME = "hello.go"; + + @Before + @Override + public void buildTokenizer() throws IOException { + this.tokenizer = new GoTokenizer(); + this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME)); + } + + @Override + public String getSampleCode() throws IOException { + return IOUtils.toString(GoTokenizer.class.getResourceAsStream(FILENAME)); + } + + @Test + public void tokenizeTest() throws IOException { + this.expectedTokenCount = 23; + super.tokenizeTest(); + } +} diff --git a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go new file mode 100644 index 0000000000..302c5a580a --- /dev/null +++ b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go @@ -0,0 +1,29 @@ +/* +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* CPD-OFF */ +package main +/* CPD-ON */ + +import ( + "fmt" + + "github.com/golang/example/stringutil" +) + +func main() { + fmt.Println(stringutil.Reverse("!selpmaxe oG ,olleH")) +} \ No newline at end of file From 16fe20af0e26284428d2682b653183595334f731 Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Mon, 28 Jan 2019 01:34:17 -0300 Subject: [PATCH 13/90] Restored imports order --- .../main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index 93fabf6049..eb9001e0d3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -4,13 +4,14 @@ package net.sourceforge.pmd.cpd; -import net.sourceforge.pmd.cpd.token.AntlrHiddenTokenFilter; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; + import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrHiddenTokenFilter; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; import static org.antlr.v4.runtime.Token.EOF; From 189bb46ea1a92087ac266409214e0f29df5182f2 Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 28 Jan 2019 14:39:08 +0530 Subject: [PATCH 14/90] UnusedImports false positive for method parameter type in @see Javadoc Updated SEE_PATTERN regex pattern. --- .../pmd/lang/java/rule/bestpractices/UnusedImportsRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UnusedImportsRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UnusedImportsRule.java index 29516d0862..402b4d14f9 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UnusedImportsRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UnusedImportsRule.java @@ -41,7 +41,7 @@ public class UnusedImportsRule extends AbstractJavaRule { * @throws package.class label */ private static final Pattern SEE_PATTERN = Pattern - .compile("@see\\s+(\\p{Alpha}\\w*)(?:#\\w*(?:\\(([\\w\\s,]*)\\))?)?"); + .compile("@see\\s+(\\p{Alpha}?+\\w*)(?:#\\w*(?:\\(([\\w\\s,]*)\\))?)?"); private static final Pattern LINK_PATTERNS = Pattern .compile("\\{@link(?:plain)?\\s+(\\p{Alpha}\\w*)(?:#\\w*(?:\\(([.\\w\\s,]*)\\))?)?[\\s\\}]"); From 1b9e63b4f492d723d4a010f0f2b2e156bac4699b Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 28 Jan 2019 16:50:51 +0530 Subject: [PATCH 15/90] #1543 [java] LinguisticNaming should ignore overriden methods Overriden methods are ignored from LinguisticNamingRule. --- .../rule/codestyle/LinguisticNamingRule.java | 22 +++++++++++++----- .../rule/codestyle/xml/LinguisticNaming.xml | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java index 061e7e4196..86edffe880 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java @@ -7,6 +7,9 @@ package net.sourceforge.pmd.lang.java.rule.codestyle; import static net.sourceforge.pmd.properties.PropertyFactory.booleanProperty; import static net.sourceforge.pmd.properties.PropertyFactory.stringListProperty; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -18,11 +21,11 @@ import net.sourceforge.pmd.lang.java.ast.ASTMethodDeclaration; import net.sourceforge.pmd.lang.java.ast.ASTResultType; import net.sourceforge.pmd.lang.java.ast.ASTType; import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclarator; -import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule; +import net.sourceforge.pmd.lang.java.rule.AbstractIgnoredAnnotationRule; import net.sourceforge.pmd.lang.java.typeresolution.TypeHelper; import net.sourceforge.pmd.properties.PropertyDescriptor; -public class LinguisticNamingRule extends AbstractJavaRule { +public class LinguisticNamingRule extends AbstractIgnoredAnnotationRule { private static final PropertyDescriptor CHECK_BOOLEAN_METHODS = booleanProperty("checkBooleanMethod").defaultValue(true).desc("Check method names and types for inconsistent naming.").build(); private static final PropertyDescriptor CHECK_GETTERS = @@ -70,6 +73,11 @@ public class LinguisticNamingRule extends AbstractJavaRule { addRuleChainVisit(ASTLocalVariableDeclaration.class); } + @Override + protected Collection defaultSuppressionAnnotations() { + return Collections.checkedList(Arrays.asList("java.lang.Override"), String.class); + } + @Override public Object visit(ASTMethodDeclaration node, Object data) { String nameOfMethod = node.getMethodName(); @@ -132,10 +140,12 @@ public class LinguisticNamingRule extends AbstractJavaRule { } private void checkSetters(ASTMethodDeclaration node, Object data, String nameOfMethod) { - ASTResultType resultType = node.getResultType(); - if (hasPrefix(nameOfMethod, "set") && !resultType.isVoid()) { - addViolationWithMessage(data, node, "Linguistics Antipattern - The setter ''{0}'' should not return any type except void linguistically", - new Object[] { nameOfMethod }); + if (!hasIgnoredAnnotation(node)) { + ASTResultType resultType = node.getResultType(); + if (hasPrefix(nameOfMethod, "set") && !resultType.isVoid()) { + addViolationWithMessage(data, node, "Linguistics Antipattern - The setter ''{0}'' should not return any type except void linguistically", + new Object[] { nameOfMethod }); + } } } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml index 006b005d35..905b91fa42 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml @@ -549,6 +549,29 @@ public class AtomicBooleanFN { AtomicBoolean willX = new AtomicBoolean(true); AtomicBoolean shouldX = new AtomicBoolean(true); } +} + ]]> + + + #1543 [java] LinguisticNaming should ignore overriden methods + 1 + From 746dc4e602ab425bd3512561cd17180e380d0e0c Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 28 Jan 2019 20:51:06 +0530 Subject: [PATCH 16/90] Code review changes All overridden methods are ignored now. --- .../rule/codestyle/LinguisticNamingRule.java | 50 +++++++++---------- .../rule/codestyle/xml/LinguisticNaming.xml | 1 + 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java index 86edffe880..e0e1cfa123 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/LinguisticNamingRule.java @@ -80,26 +80,28 @@ public class LinguisticNamingRule extends AbstractIgnoredAnnotationRule { @Override public Object visit(ASTMethodDeclaration node, Object data) { - String nameOfMethod = node.getMethodName(); + if (!hasIgnoredAnnotation(node)) { + String nameOfMethod = node.getMethodName(); - if (getProperty(CHECK_BOOLEAN_METHODS)) { - checkBooleanMethods(node, data, nameOfMethod); - } - - if (getProperty(CHECK_SETTERS)) { - checkSetters(node, data, nameOfMethod); - } - - if (getProperty(CHECK_GETTERS)) { - checkGetters(node, data, nameOfMethod); - } - - if (getProperty(CHECK_PREFIXED_TRANSFORM_METHODS)) { - checkPrefixedTransformMethods(node, data, nameOfMethod); - } - - if (getProperty(CHECK_TRANSFORM_METHODS)) { - checkTransformMethods(node, data, nameOfMethod); + if (getProperty(CHECK_BOOLEAN_METHODS)) { + checkBooleanMethods(node, data, nameOfMethod); + } + + if (getProperty(CHECK_SETTERS)) { + checkSetters(node, data, nameOfMethod); + } + + if (getProperty(CHECK_GETTERS)) { + checkGetters(node, data, nameOfMethod); + } + + if (getProperty(CHECK_PREFIXED_TRANSFORM_METHODS)) { + checkPrefixedTransformMethods(node, data, nameOfMethod); + } + + if (getProperty(CHECK_TRANSFORM_METHODS)) { + checkTransformMethods(node, data, nameOfMethod); + } } return data; @@ -140,12 +142,10 @@ public class LinguisticNamingRule extends AbstractIgnoredAnnotationRule { } private void checkSetters(ASTMethodDeclaration node, Object data, String nameOfMethod) { - if (!hasIgnoredAnnotation(node)) { - ASTResultType resultType = node.getResultType(); - if (hasPrefix(nameOfMethod, "set") && !resultType.isVoid()) { - addViolationWithMessage(data, node, "Linguistics Antipattern - The setter ''{0}'' should not return any type except void linguistically", - new Object[] { nameOfMethod }); - } + ASTResultType resultType = node.getResultType(); + if (hasPrefix(nameOfMethod, "set") && !resultType.isVoid()) { + addViolationWithMessage(data, node, "Linguistics Antipattern - The setter ''{0}'' should not return any type except void linguistically", + new Object[] { nameOfMethod }); } } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml index 905b91fa42..f75c8a8d2e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/LinguisticNaming.xml @@ -555,6 +555,7 @@ public class AtomicBooleanFN { #1543 [java] LinguisticNaming should ignore overriden methods 1 + 4 Date: Mon, 28 Jan 2019 23:02:59 +0530 Subject: [PATCH 17/90] Test cases added Updated testcases. --- .../rule/bestpractices/unusedimports/Base.java | 10 ++++++++++ .../rule/bestpractices/xml/UnusedImports.xml | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java new file mode 100644 index 0000000000..bd90539900 --- /dev/null +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java @@ -0,0 +1,10 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.java.rule.bestpractices.unusedimports; + +public class Base { + void method(java.io.File file) { + } +} diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml index 223783eb2c..10fe98740e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml @@ -452,6 +452,23 @@ public class Imports { i = g3(i); return i; } +} + ]]> + + + + #1625 [java] UnusedImports false positive for method parameter type in @see Javadoc + 0 + From c76c9734e0ef8fd8c8158dd23e9d0c280fd87b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Mon, 28 Jan 2019 23:46:03 -0300 Subject: [PATCH 18/90] Update changelog, refs #1623 --- docs/pages/release_notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..1e215b2cd9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,9 +16,14 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java-codestyle + * [#1547](https://github.com/pmd/pmd/issues/1547): \[java] AtLeastOneConstructorRule: false-positive with lombok.AllArgsConstructor + ### API Changes ### External Contributions +* [#1623](https://github.com/pmd/pmd/pull/1623): \[java] Fix lombok.AllArgsConstructor support - [Bobby Wertman](https://github.com/CasualSuperman) + {% endtocmaker %} From 081ee17349c512783a3f0b0e8be762d87abac4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Mon, 28 Jan 2019 23:59:19 -0300 Subject: [PATCH 19/90] Simplify test --- .../java/rule/bestpractices/unusedimports/Base.java | 10 ---------- .../lang/java/rule/bestpractices/xml/UnusedImports.xml | 7 +++---- 2 files changed, 3 insertions(+), 14 deletions(-) delete mode 100644 pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java deleted file mode 100644 index bd90539900..0000000000 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/unusedimports/Base.java +++ /dev/null @@ -1,10 +0,0 @@ -/** - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.java.rule.bestpractices.unusedimports; - -public class Base { - void method(java.io.File file) { - } -} diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml index 10fe98740e..e01caa625e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedImports.xml @@ -462,12 +462,11 @@ public class Imports { From 212fd100aacb8ed00a5353382c996450dc8c0bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Mon, 28 Jan 2019 23:59:30 -0300 Subject: [PATCH 20/90] Update changelog, refs #1625 --- docs/pages/release_notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..56e15ca886 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,9 +16,14 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java-bestpractices + * [#1555](https://github.com/pmd/pmd/issues/1555): \[java] UnusedImports false positive for method parameter type in @see Javadoc + ### API Changes ### External Contributions +* [#1625](https://github.com/pmd/pmd/pull/1625): \[java] UnusedImports false positive for method parameter type in @see Javadoc - [Shubham](https://github.com/Shubham-2k17) + {% endtocmaker %} From 754a812ad2d9bb7c3f66659c4b6cee6d738c1e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Tue, 29 Jan 2019 00:03:44 -0300 Subject: [PATCH 21/90] Update changelog, refs #1628 --- docs/pages/release_notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..3aa27c29bb 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,9 +16,14 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java-codestyle + * [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods + ### API Changes ### External Contributions +* [#1628](https://github.com/pmd/pmd/pull/1628): \[java] LinguisticNaming should ignore overriden methods - [Shubham](https://github.com/Shubham-2k17) + {% endtocmaker %} From dab1fb24c6fe982d0761d3daebecd4d223697c15 Mon Sep 17 00:00:00 2001 From: "Travis CI (pmd-bot)" Date: Tue, 29 Jan 2019 03:22:02 +0000 Subject: [PATCH 22/90] Update documentation TRAVIS_JOB_NUMBER=3376.1 TRAVIS_COMMIT_RANGE=add4a78b57a2...8a7913a30d93 --- docs/pages/pmd/rules/java/codestyle.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/pages/pmd/rules/java/codestyle.md b/docs/pages/pmd/rules/java/codestyle.md index 0d74b9cebc..9ab0b79baa 100644 --- a/docs/pages/pmd/rules/java/codestyle.md +++ b/docs/pages/pmd/rules/java/codestyle.md @@ -77,7 +77,7 @@ public class Foo { |Name|Default Value|Description|Multivalued| |----|-------------|-----------|-----------| -|ignoredAnnotations|lombok.Data \| lombok.Value \| lombok.Builder \| lombok.NoArgsConstructor \| lombok.RequiredArgsConstructor \| lombok.AllArgsConstructorAtLeastOneConstructor|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.| +|ignoredAnnotations|lombok.Data \| lombok.Value \| lombok.Builder \| lombok.NoArgsConstructor \| lombok.RequiredArgsConstructor \| lombok.AllArgsConstructor|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.| **Use this rule by referencing it:** ``` xml @@ -1160,6 +1160,7 @@ public class LinguisticNaming { |Name|Default Value|Description|Multivalued| |----|-------------|-----------|-----------| +|ignoredAnnotations|java.lang.Override|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.| |checkBooleanMethod|true|Check method names and types for inconsistent naming.|no| |checkGetters|true|Check return type of getters.|no| |checkSetters|true|Check return type of setters.|no| From 9532552eaaad564b4aabbb70b4f663b53f5f5f00 Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 30 Jan 2019 11:09:44 +0530 Subject: [PATCH 23/90] BeanMembersShouldSerializeRule aware of lombok accessors Extended AbstractLombokAwareRule to make BeanMembersShouldSerializeRule aware of lombok annotations. Checking if class or any of the variables have lombok annotations and avoid BeanMembersShouldSerializeRule on these variables. --- .../BeanMembersShouldSerializeRule.java | 10 ++- .../xml/BeanMembersShouldSerialize.xml | 78 +++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java index b36674a30e..9b16bf3c5c 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java @@ -21,14 +21,15 @@ import net.sourceforge.pmd.lang.java.ast.ASTMethodDeclarator; import net.sourceforge.pmd.lang.java.ast.ASTPrimitiveType; import net.sourceforge.pmd.lang.java.ast.ASTResultType; import net.sourceforge.pmd.lang.java.ast.AccessNode; -import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule; +import net.sourceforge.pmd.lang.java.ast.Annotatable; +import net.sourceforge.pmd.lang.java.rule.AbstractLombokAwareRule; import net.sourceforge.pmd.lang.java.symboltable.ClassScope; import net.sourceforge.pmd.lang.java.symboltable.MethodNameDeclaration; import net.sourceforge.pmd.lang.java.symboltable.VariableNameDeclaration; import net.sourceforge.pmd.lang.symboltable.NameOccurrence; import net.sourceforge.pmd.properties.PropertyDescriptor; -public class BeanMembersShouldSerializeRule extends AbstractJavaRule { +public class BeanMembersShouldSerializeRule extends AbstractLombokAwareRule { private String prefixProperty; @@ -62,6 +63,8 @@ public class BeanMembersShouldSerializeRule extends AbstractJavaRule { return data; } + boolean classHasLombok = hasLombokAnnotation(node); + Map> methods = node.getScope().getEnclosingScope(ClassScope.class) .getMethodDeclarations(); List getSetMethList = new ArrayList<>(methods.size()); @@ -81,7 +84,8 @@ public class BeanMembersShouldSerializeRule extends AbstractJavaRule { for (Map.Entry> entry : vars.entrySet()) { VariableNameDeclaration decl = entry.getKey(); AccessNode accessNodeParent = decl.getAccessNodeParent(); - if (entry.getValue().isEmpty() || accessNodeParent.isTransient() || accessNodeParent.isStatic()) { + if (entry.getValue().isEmpty() || accessNodeParent.isTransient() || accessNodeParent.isStatic() + || classHasLombok || hasIgnoredAnnotation((Annotatable) accessNodeParent)) { continue; } String varName = StringUtils.capitalize(trimIfPrefix(decl.getImage())); diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/BeanMembersShouldSerialize.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/BeanMembersShouldSerialize.xml index 90b4112ad4..983fca10cd 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/BeanMembersShouldSerialize.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/BeanMembersShouldSerialize.xml @@ -185,6 +185,84 @@ public class Foo { private String baz; public void setBaz(String b) { baz = b; } public String getBaz() { return baz; } +} + ]]> + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 1 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 2 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 3 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 4 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 5 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 6 + 0 + + + + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 7 + 0 + From 149f85a7e417829e325aa68f619d169223d6599e Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 30 Jan 2019 14:02:17 +0530 Subject: [PATCH 24/90] #1633 [java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat Thread safe formatter org.apache.commons.lang3.time.FastDateFormat can not avoid UnsynchronizedStaticFormatterRule. --- .../UnsynchronizedStaticFormatterRule.java | 7 +++++++ .../xml/UnsynchronizedStaticFormatter.xml | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java index f53089abcb..4402b17ba8 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java @@ -5,6 +5,7 @@ package net.sourceforge.pmd.lang.java.rule.multithreading; import java.text.Format; +import java.util.Arrays; import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType; @@ -27,6 +28,9 @@ import net.sourceforge.pmd.lang.symboltable.NameOccurrence; */ public class UnsynchronizedStaticFormatterRule extends AbstractJavaRule { private Class formatterClassToCheck = Format.class; + private String[] threadSafeFormatter = { + "org.apache.commons.lang3.time.FastDateFormat", + }; public UnsynchronizedStaticFormatterRule() { addRuleChainVisit(ASTFieldDeclaration.class); @@ -48,6 +52,9 @@ public class UnsynchronizedStaticFormatterRule extends AbstractJavaRule { } ASTVariableDeclaratorId var = node.getFirstDescendantOfType(ASTVariableDeclaratorId.class); + if (Arrays.asList(threadSafeFormatter).contains(var.getType().getName())) { + return data; + } for (NameOccurrence occ : var.getUsages()) { Node n = occ.getLocation(); if (n.getFirstParentOfType(ASTSynchronizedStatement.class) != null) { diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/multithreading/xml/UnsynchronizedStaticFormatter.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/multithreading/xml/UnsynchronizedStaticFormatter.xml index 365c79fd99..5c9db4e1f0 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/multithreading/xml/UnsynchronizedStaticFormatter.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/multithreading/xml/UnsynchronizedStaticFormatter.xml @@ -132,6 +132,20 @@ public class Foo { void bar() { messageFormat.format(); } +} + ]]> + + + #1633 [java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat + 0 + From d1c75ccff4d584a2ef7a6d819ce98b8e64dd155d Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 30 Jan 2019 14:57:39 +0530 Subject: [PATCH 25/90] Unused Formatter need not be checked for synchronization. --- .../rule/multithreading/UnsynchronizedStaticFormatterRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java index 4402b17ba8..2a1f6b386b 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java @@ -52,7 +52,7 @@ public class UnsynchronizedStaticFormatterRule extends AbstractJavaRule { } ASTVariableDeclaratorId var = node.getFirstDescendantOfType(ASTVariableDeclaratorId.class); - if (Arrays.asList(threadSafeFormatter).contains(var.getType().getName())) { + if (var.getType() != null && Arrays.asList(threadSafeFormatter).contains(var.getType().getName())) { return data; } for (NameOccurrence occ : var.getUsages()) { From cb0c70f1aca2491474fe581df1b6fe729a63cef5 Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 30 Jan 2019 18:58:47 +0530 Subject: [PATCH 26/90] Review changes implemented --- .../BeanMembersShouldSerializeRule.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java index 9b16bf3c5c..910101c31a 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/BeanMembersShouldSerializeRule.java @@ -8,6 +8,7 @@ import static net.sourceforge.pmd.properties.PropertyFactory.stringProperty; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -35,11 +36,19 @@ public class BeanMembersShouldSerializeRule extends AbstractLombokAwareRule { private static final PropertyDescriptor PREFIX_DESCRIPTOR = stringProperty("prefix").desc("A variable prefix to skip, i.e., m_").defaultValue("").build(); - public BeanMembersShouldSerializeRule() { definePropertyDescriptor(PREFIX_DESCRIPTOR); } + @Override + protected Collection defaultSuppressionAnnotations() { + return Arrays.asList( + "lombok.Data", + "lombok.Getter", + "lombok.Value" + ); + } + @Override public Object visit(ASTCompilationUnit node, Object data) { prefixProperty = getProperty(PREFIX_DESCRIPTOR); @@ -63,7 +72,9 @@ public class BeanMembersShouldSerializeRule extends AbstractLombokAwareRule { return data; } - boolean classHasLombok = hasLombokAnnotation(node); + if (hasLombokAnnotation(node)) { + return super.visit(node, data); + } Map> methods = node.getScope().getEnclosingScope(ClassScope.class) .getMethodDeclarations(); @@ -85,7 +96,7 @@ public class BeanMembersShouldSerializeRule extends AbstractLombokAwareRule { VariableNameDeclaration decl = entry.getKey(); AccessNode accessNodeParent = decl.getAccessNodeParent(); if (entry.getValue().isEmpty() || accessNodeParent.isTransient() || accessNodeParent.isStatic() - || classHasLombok || hasIgnoredAnnotation((Annotatable) accessNodeParent)) { + || hasIgnoredAnnotation((Annotatable) accessNodeParent)) { continue; } String varName = StringUtils.capitalize(trimIfPrefix(decl.getImage())); From 054789793413a1aaddeef4edf6a05f4b6aa9186a Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 30 Jan 2019 19:43:09 +0530 Subject: [PATCH 27/90] Reviewed code updated --- .../UnsynchronizedStaticFormatterRule.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java index 2a1f6b386b..e4f43f3ecb 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/multithreading/UnsynchronizedStaticFormatterRule.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.lang.java.rule.multithreading; import java.text.Format; import java.util.Arrays; +import java.util.List; import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType; @@ -28,9 +29,9 @@ import net.sourceforge.pmd.lang.symboltable.NameOccurrence; */ public class UnsynchronizedStaticFormatterRule extends AbstractJavaRule { private Class formatterClassToCheck = Format.class; - private String[] threadSafeFormatter = { - "org.apache.commons.lang3.time.FastDateFormat", - }; + private static final List THREAD_SAFE_FORMATTER = Arrays.asList( + "org.apache.commons.lang3.time.FastDateFormat" + ); public UnsynchronizedStaticFormatterRule() { addRuleChainVisit(ASTFieldDeclaration.class); @@ -52,8 +53,10 @@ public class UnsynchronizedStaticFormatterRule extends AbstractJavaRule { } ASTVariableDeclaratorId var = node.getFirstDescendantOfType(ASTVariableDeclaratorId.class); - if (var.getType() != null && Arrays.asList(threadSafeFormatter).contains(var.getType().getName())) { - return data; + for (String formatter: THREAD_SAFE_FORMATTER) { + if (TypeHelper.isA(var, formatter)) { + return data; + } } for (NameOccurrence occ : var.getUsages()) { Node n = occ.getLocation(); From 0703b041a8a0f8c50a109cdaf272855e70aa7f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 30 Jan 2019 12:15:53 -0300 Subject: [PATCH 28/90] Update changelog, refs #1635 --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 949fdcced5..fcf470433e 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -21,6 +21,8 @@ This is a {{ site.pmd.release_type }} release. * java-codestyle * [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods * [#1547](https://github.com/pmd/pmd/issues/1547): \[java] AtLeastOneConstructorRule: false-positive with lombok.AllArgsConstructor +* java-multithreading + * [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat ### API Changes @@ -29,6 +31,7 @@ This is a {{ site.pmd.release_type }} release. * [#1623](https://github.com/pmd/pmd/pull/1623): \[java] Fix lombok.AllArgsConstructor support - [Bobby Wertman](https://github.com/CasualSuperman) * [#1625](https://github.com/pmd/pmd/pull/1625): \[java] UnusedImports false positive for method parameter type in @see Javadoc - [Shubham](https://github.com/Shubham-2k17) * [#1628](https://github.com/pmd/pmd/pull/1628): \[java] LinguisticNaming should ignore overriden methods - [Shubham](https://github.com/Shubham-2k17) +* [#1635](https://github.com/pmd/pmd/pull/1635): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat - [Shubham](https://github.com/Shubham-2k17) {% endtocmaker %} From db3d1c8362432b1aace1bf9cea418043738f71e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 30 Jan 2019 12:26:17 -0300 Subject: [PATCH 29/90] Update changelog, refs #1634 --- docs/pages/release_notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..4fbce11087 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,9 +16,14 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java-errorprone + * [#780](https://github.com/pmd/pmd/issues/780): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors + ### API Changes ### External Contributions +* [#1634](https://github.com/pmd/pmd/pull/1634): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors - [Shubham](https://github.com/Shubham-2k17) + {% endtocmaker %} From bd7c6c3a17f769ba1fc548578fb42fca824de1c2 Mon Sep 17 00:00:00 2001 From: "Travis CI (pmd-bot)" Date: Wed, 30 Jan 2019 15:45:01 +0000 Subject: [PATCH 30/90] Update documentation TRAVIS_JOB_NUMBER=3387.1 TRAVIS_COMMIT_RANGE=0ae9ed045b8e...e2a4b744fd54 --- docs/pages/pmd/rules/java/errorprone.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/pmd/rules/java/errorprone.md b/docs/pages/pmd/rules/java/errorprone.md index 173243fa7a..1f5f84cc15 100644 --- a/docs/pages/pmd/rules/java/errorprone.md +++ b/docs/pages/pmd/rules/java/errorprone.md @@ -738,6 +738,7 @@ private int getMoreFoo(){ |Name|Default Value|Description|Multivalued| |----|-------------|-----------|-----------| +|ignoredAnnotations|lombok.Data \| lombok.Getter \| lombok.Value|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.| |prefix||A variable prefix to skip, i.e., m\_|no| **Use this rule by referencing it:** From 7a5ed78a73cfa58eb7ad8f255f733a873feec3e0 Mon Sep 17 00:00:00 2001 From: Shubham Date: Thu, 31 Jan 2019 15:40:22 +0530 Subject: [PATCH 31/90] constants initialized by literals avoided by AccessorMethodGenerationRule Compile time constants with literals only expression avoiding AccessorMethodGenerationRule. --- .../AccessorMethodGenerationRule.java | 21 ++++++++++++ .../xml/AccessorMethodGeneration.xml | 33 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java index 56cad28680..30f9dee2b0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java @@ -10,6 +10,9 @@ import java.util.Map; import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit; import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration; import net.sourceforge.pmd.lang.java.ast.ASTMethodDeclaration; +import net.sourceforge.pmd.lang.java.ast.ASTPrimaryPrefix; +import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclarator; +import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer; import net.sourceforge.pmd.lang.java.ast.AbstractJavaAccessNode; import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule; import net.sourceforge.pmd.lang.java.symboltable.AbstractJavaScope; @@ -18,6 +21,7 @@ import net.sourceforge.pmd.lang.java.symboltable.ClassScope; import net.sourceforge.pmd.lang.java.symboltable.MethodNameDeclaration; import net.sourceforge.pmd.lang.java.symboltable.SourceFileScope; import net.sourceforge.pmd.lang.java.symboltable.VariableNameDeclaration; +import net.sourceforge.pmd.lang.java.typeresolution.TypeHelper; import net.sourceforge.pmd.lang.symboltable.NameOccurrence; public class AccessorMethodGenerationRule extends AbstractJavaRule { @@ -57,6 +61,23 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { return; } + for (final ASTVariableDeclarator varDecl: node.findChildrenOfType(ASTVariableDeclarator.class)) { + if (varDecl.hasInitializer()) { + ASTVariableInitializer varInit = varDecl.getInitializer(); + List initPrefix = varInit.findDescendantsOfType(ASTPrimaryPrefix.class); + boolean allLiterals = true; + for (ASTPrimaryPrefix prefix: initPrefix) { + if (!prefix.getType().isPrimitive() && !TypeHelper.isA(prefix, "java.lang.String")) { + allLiterals = false; + break; + } + } + if (allLiterals) { + return; + } + } + } + for (final NameOccurrence no : occurrences) { ClassScope usedAtScope = no.getLocation().getScope().getEnclosingScope(ClassScope.class); diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml index bf1b8fd638..2bf6408f9a 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml @@ -190,6 +190,39 @@ public class Foo implements Parcelable { return value == 1; } }; +} + ]]> + java 10 + + + #808 - [java] AccessorMethodGeneration false positives with compile time constants - literals + 1 + java 10 From 23ce8435e4f367d90ad8f9d3132a6df802b9e2a8 Mon Sep 17 00:00:00 2001 From: Shubham Date: Fri, 1 Feb 2019 19:36:51 +0530 Subject: [PATCH 32/90] all compile time constants are avoided by AccessorMethodGenerationRule --- .../AccessorMethodGenerationRule.java | 60 ++++++++++++++++--- .../xml/AccessorMethodGeneration.xml | 13 +++- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java index 30f9dee2b0..d9f3ccf823 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java @@ -4,13 +4,18 @@ package net.sourceforge.pmd.lang.java.rule.bestpractices; +import java.util.HashMap; import java.util.List; import java.util.Map; import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit; +import net.sourceforge.pmd.lang.java.ast.ASTExpression; import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration; +import net.sourceforge.pmd.lang.java.ast.ASTLiteral; import net.sourceforge.pmd.lang.java.ast.ASTMethodDeclaration; -import net.sourceforge.pmd.lang.java.ast.ASTPrimaryPrefix; +import net.sourceforge.pmd.lang.java.ast.ASTName; +import net.sourceforge.pmd.lang.java.ast.ASTPrimaryExpression; +import net.sourceforge.pmd.lang.java.ast.ASTPrimarySuffix; import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclarator; import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer; import net.sourceforge.pmd.lang.java.ast.AbstractJavaAccessNode; @@ -21,11 +26,12 @@ import net.sourceforge.pmd.lang.java.symboltable.ClassScope; import net.sourceforge.pmd.lang.java.symboltable.MethodNameDeclaration; import net.sourceforge.pmd.lang.java.symboltable.SourceFileScope; import net.sourceforge.pmd.lang.java.symboltable.VariableNameDeclaration; -import net.sourceforge.pmd.lang.java.typeresolution.TypeHelper; import net.sourceforge.pmd.lang.symboltable.NameOccurrence; public class AccessorMethodGenerationRule extends AbstractJavaRule { + private Map cache = new HashMap<>(); + @Override public Object visit(final ASTCompilationUnit node, final Object data) { final SourceFileScope file = node.getScope().getEnclosingScope(SourceFileScope.class); @@ -64,17 +70,24 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { for (final ASTVariableDeclarator varDecl: node.findChildrenOfType(ASTVariableDeclarator.class)) { if (varDecl.hasInitializer()) { ASTVariableInitializer varInit = varDecl.getInitializer(); - List initPrefix = varInit.findDescendantsOfType(ASTPrimaryPrefix.class); - boolean allLiterals = true; - for (ASTPrimaryPrefix prefix: initPrefix) { - if (!prefix.getType().isPrimitive() && !TypeHelper.isA(prefix, "java.lang.String")) { - allLiterals = false; - break; + List initExpression = varInit.findDescendantsOfType(ASTExpression.class); + boolean isConstantExpression = true; + constantCheck: + for (ASTExpression exp: initExpression) { + List primaryExpressions = exp.findDescendantsOfType(ASTPrimaryExpression.class); + for (ASTPrimaryExpression expression: primaryExpressions) { + if (!isCompileTimeConstant(expression)) { + isConstantExpression = false; + break constantCheck; + } } } - if (allLiterals) { + cache.put(varDecl.getName(), isConstantExpression); + if (isConstantExpression) { return; } + } else { + cache.put(varDecl.getName(), false); } } @@ -87,4 +100,33 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { } } } + + public boolean isCompileTimeConstant(ASTPrimaryExpression expressions) { + // function call detected + List suffix = expressions.findDescendantsOfType(ASTPrimarySuffix.class); + if (!suffix.isEmpty()) { + return false; + } + + // single node expression + List nameNodes = expressions.findDescendantsOfType(ASTName.class); + List literalNodes = expressions.findDescendantsOfType(ASTLiteral.class); + if (nameNodes.size() + literalNodes.size() < 2) { + for (ASTName node: nameNodes) { + if (cache.containsKey(node.getImage())) { + return cache.get(node.getImage()); + } + } + return true; + } + + // multiple node expression + List subExpressions = expressions.findDescendantsOfType(ASTPrimaryExpression.class); + for (ASTPrimaryExpression exp: subExpressions) { + if (!isCompileTimeConstant(exp)) { + return false; + } + } + return true; + } } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml index 2bf6408f9a..1520bde45a 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml @@ -195,8 +195,9 @@ public class Foo implements Parcelable { java 10 - #808 - [java] AccessorMethodGeneration false positives with compile time constants - literals - 1 + #808 - [java] AccessorMethodGeneration false positives with compile time constants + 3 + 24,27,28 Date: Fri, 1 Feb 2019 11:02:42 +0100 Subject: [PATCH 33/90] Update instead of override classHasLombokAnnotation flag Fixes issues where inner classes would override this flag. --- .../java/rule/AbstractLombokAwareRule.java | 12 ++++-- .../java/rule/design/ImmutableFieldRule.java | 2 +- .../java/rule/design/xml/SingularField.xml | 41 +++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/AbstractLombokAwareRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/AbstractLombokAwareRule.java index 710f374aee..e0f8fd5345 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/AbstractLombokAwareRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/AbstractLombokAwareRule.java @@ -65,21 +65,27 @@ public class AbstractLombokAwareRule extends AbstractIgnoredAnnotationRule { @Override public Object visit(ASTClassOrInterfaceDeclaration node, Object data) { + boolean oldValue = classHasLombokAnnotation; classHasLombokAnnotation = hasLombokAnnotation(node); - return super.visit(node, data); + Object result = super.visit(node, data); + classHasLombokAnnotation = oldValue; + return result; } @Override public Object visit(ASTEnumDeclaration node, Object data) { + boolean oldValue = classHasLombokAnnotation; classHasLombokAnnotation = hasLombokAnnotation(node); - return super.visit(node, data); + Object result = super.visit(node, data); + classHasLombokAnnotation = oldValue; + return result; } /** * Returns whether there have been class level Lombok annotations found. * Note: this can only be queried after the class declaration node has been * processed. - * + * * @return true if a lombok annotation at the class level has * been found */ diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/design/ImmutableFieldRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/design/ImmutableFieldRule.java index d409c66d90..a4385b640f 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/design/ImmutableFieldRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/design/ImmutableFieldRule.java @@ -55,7 +55,7 @@ public class ImmutableFieldRule extends AbstractLombokAwareRule { AccessNode accessNodeParent = field.getAccessNodeParent(); if (accessNodeParent.isStatic() || !accessNodeParent.isPrivate() || accessNodeParent.isFinal() || accessNodeParent.isVolatile() - || hasClassLombokAnnotation() + || hasLombokAnnotation(node) || hasIgnoredAnnotation((Annotatable) accessNodeParent)) { continue; } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/SingularField.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/SingularField.xml index 5b9ff1086a..4caef8662a 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/SingularField.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/SingularField.xml @@ -573,6 +573,47 @@ public class MyClass { ]]> + + [java] SingularField: Lombok false positive with inner class and annotated outer class + 0 + + + + + [java] SingularField: Lombok false positive with annotated inner class + 1 + + #177 [java] SingularField with lambdas as final fields From 6d17dc2e45f047d88df30b679f5dfae43a6ba776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Fri, 1 Feb 2019 19:25:35 -0300 Subject: [PATCH 34/90] Update changelog, refs #1640 --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index bbe45d6b90..a9d8201b0a 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -21,6 +21,8 @@ This is a {{ site.pmd.release_type }} release. * java-codestyle * [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods * [#1547](https://github.com/pmd/pmd/issues/1547): \[java] AtLeastOneConstructorRule: false-positive with lombok.AllArgsConstructor +* java-design + * [#1641](https://github.com/pmd/pmd/issues/1641): \[java] False-positive with Lombok and inner classes * java-errorprone * [#780](https://github.com/pmd/pmd/issues/780): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors * java-multithreading @@ -35,6 +37,7 @@ This is a {{ site.pmd.release_type }} release. * [#1628](https://github.com/pmd/pmd/pull/1628): \[java] LinguisticNaming should ignore overriden methods - [Shubham](https://github.com/Shubham-2k17) * [#1634](https://github.com/pmd/pmd/pull/1634): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors - [Shubham](https://github.com/Shubham-2k17) * [#1635](https://github.com/pmd/pmd/pull/1635): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat - [Shubham](https://github.com/Shubham-2k17) +* [#1640](https://github.com/pmd/pmd/pull/1640): \[java] Update instead of override classHasLombokAnnotation flag - [Phokham Nonava](https://github.com/fluxroot) {% endtocmaker %} From 1e1d6aafae687fc5da22a86bb1d46147b2088491 Mon Sep 17 00:00:00 2001 From: Shubham Date: Sat, 2 Feb 2019 20:38:23 +0530 Subject: [PATCH 35/90] code review changes --- .../AccessorMethodGenerationRule.java | 50 +++++++++++-------- .../xml/AccessorMethodGeneration.xml | 10 ++-- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java index d9f3ccf823..8a43087166 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java @@ -4,10 +4,11 @@ package net.sourceforge.pmd.lang.java.rule.bestpractices; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import net.sourceforge.pmd.RuleContext; import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit; import net.sourceforge.pmd.lang.java.ast.ASTExpression; import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration; @@ -30,7 +31,7 @@ import net.sourceforge.pmd.lang.symboltable.NameOccurrence; public class AccessorMethodGenerationRule extends AbstractJavaRule { - private Map cache = new HashMap<>(); + private List cache = new ArrayList<>(); @Override public Object visit(final ASTCompilationUnit node, final Object data) { @@ -67,27 +68,27 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { return; } - for (final ASTVariableDeclarator varDecl: node.findChildrenOfType(ASTVariableDeclarator.class)) { - if (varDecl.hasInitializer()) { - ASTVariableInitializer varInit = varDecl.getInitializer(); - List initExpression = varInit.findDescendantsOfType(ASTExpression.class); - boolean isConstantExpression = true; - constantCheck: - for (ASTExpression exp: initExpression) { - List primaryExpressions = exp.findDescendantsOfType(ASTPrimaryExpression.class); - for (ASTPrimaryExpression expression: primaryExpressions) { - if (!isCompileTimeConstant(expression)) { - isConstantExpression = false; - break constantCheck; + if (node.isFinal()) { + for (final ASTVariableDeclarator varDecl: node.findChildrenOfType(ASTVariableDeclarator.class)) { + if (varDecl.hasInitializer()) { + ASTVariableInitializer varInit = varDecl.getInitializer(); + List initExpression = varInit.findDescendantsOfType(ASTExpression.class); + boolean isConstantExpression = true; + constantCheck: + for (ASTExpression exp: initExpression) { + List primaryExpressions = exp.findDescendantsOfType(ASTPrimaryExpression.class); + for (ASTPrimaryExpression expression: primaryExpressions) { + if (!isCompileTimeConstant(expression)) { + isConstantExpression = false; + break constantCheck; + } } } + if (isConstantExpression) { + cache.add(varDecl.getName()); + return; + } } - cache.put(varDecl.getName(), isConstantExpression); - if (isConstantExpression) { - return; - } - } else { - cache.put(varDecl.getName(), false); } } @@ -113,8 +114,8 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { List literalNodes = expressions.findDescendantsOfType(ASTLiteral.class); if (nameNodes.size() + literalNodes.size() < 2) { for (ASTName node: nameNodes) { - if (cache.containsKey(node.getImage())) { - return cache.get(node.getImage()); + if (!cache.contains(node.getImage())) { + return false; } } return true; @@ -129,4 +130,9 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { } return true; } + + @Override + public void end(RuleContext ctx) { + cache.clear(); + } } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml index 1520bde45a..b053b9a31e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AccessorMethodGeneration.xml @@ -196,8 +196,8 @@ public class Foo implements Parcelable { #808 - [java] AccessorMethodGeneration false positives with compile time constants - 3 - 24,27,28 + 4 + 25,26,27,28 Date: Sat, 2 Feb 2019 13:56:29 -0300 Subject: [PATCH 36/90] Add todo for follow up improvements --- .../java/rule/bestpractices/AccessorMethodGenerationRule.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java index 8a43087166..b84491331d 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/bestpractices/AccessorMethodGenerationRule.java @@ -114,6 +114,7 @@ public class AccessorMethodGenerationRule extends AbstractJavaRule { List literalNodes = expressions.findDescendantsOfType(ASTLiteral.class); if (nameNodes.size() + literalNodes.size() < 2) { for (ASTName node: nameNodes) { + // TODO : use the symbol table to get the declaration of the referenced var and check it if (!cache.contains(node.getImage())) { return false; } From f1c0afddab47bbad384b14fb47b0bb2933ed153f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sat, 2 Feb 2019 13:58:00 -0300 Subject: [PATCH 37/90] Update changelog, refs #1637 --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 949fdcced5..b28919ed88 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -17,6 +17,7 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues * java-bestpractices + * [#808](https://github.com/pmd/pmd/issues/808): \[java] AccessorMethodGeneration false positives with compile time constants * [#1555](https://github.com/pmd/pmd/issues/1555): \[java] UnusedImports false positive for method parameter type in @see Javadoc * java-codestyle * [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods @@ -29,6 +30,7 @@ This is a {{ site.pmd.release_type }} release. * [#1623](https://github.com/pmd/pmd/pull/1623): \[java] Fix lombok.AllArgsConstructor support - [Bobby Wertman](https://github.com/CasualSuperman) * [#1625](https://github.com/pmd/pmd/pull/1625): \[java] UnusedImports false positive for method parameter type in @see Javadoc - [Shubham](https://github.com/Shubham-2k17) * [#1628](https://github.com/pmd/pmd/pull/1628): \[java] LinguisticNaming should ignore overriden methods - [Shubham](https://github.com/Shubham-2k17) +* [#1637](https://github.com/pmd/pmd/pull/1637): \[java] Compile time constants initialized by literals avoided by AccessorMethodGenerationRule - [Shubham](https://github.com/Shubham-2k17) {% endtocmaker %} From 295c945978df16796a791ea5fbe9184e97a354a2 Mon Sep 17 00:00:00 2001 From: tomdaly Date: Sun, 3 Feb 2019 19:11:37 +0000 Subject: [PATCH 38/90] Add property to allow apex test methods to contain underscores Add a 'skipTestMethodUnderscore' boolean property that allows an annotated Apex test method (@isTest or testMethod) to contain underscores Fixes issue #1573 --- docs/pages/pmd/rules/apex/codestyle.md | 1 + .../MethodNamingConventionsRule.java | 19 +++++++++++++++++++ .../codestyle/xml/MethodNamingConventions.xml | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/docs/pages/pmd/rules/apex/codestyle.md b/docs/pages/pmd/rules/apex/codestyle.md index a158925f87..2db40d95b6 100644 --- a/docs/pages/pmd/rules/apex/codestyle.md +++ b/docs/pages/pmd/rules/apex/codestyle.md @@ -187,6 +187,7 @@ public class Foo { |cc\_categories|Style|Code Climate Categories|yes. Delimiter is '\|'.| |cc\_remediation\_points\_multiplier|1|Code Climate Remediation Points multiplier|no| |cc\_block\_highlighting|false|Code Climate Block Highlighting|no| +|skipTestMethodUnderscores|false|Skip underscores in test methods|no| **Use this rule by referencing it:** ``` xml diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/codestyle/MethodNamingConventionsRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/codestyle/MethodNamingConventionsRule.java index 7192e57900..6041c07c18 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/codestyle/MethodNamingConventionsRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/codestyle/MethodNamingConventionsRule.java @@ -5,15 +5,26 @@ package net.sourceforge.pmd.lang.apex.rule.codestyle; import static apex.jorje.semantic.symbol.type.ModifierTypeInfos.OVERRIDE; +import static net.sourceforge.pmd.properties.PropertyFactory.booleanProperty; import net.sourceforge.pmd.lang.apex.ast.ASTMethod; +import net.sourceforge.pmd.lang.apex.ast.ASTModifierNode; import net.sourceforge.pmd.lang.apex.ast.ASTProperty; import net.sourceforge.pmd.lang.apex.ast.ASTUserClass; import net.sourceforge.pmd.lang.apex.rule.AbstractApexRule; +import net.sourceforge.pmd.properties.PropertyDescriptor; public class MethodNamingConventionsRule extends AbstractApexRule { + private static final PropertyDescriptor SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR + = booleanProperty("skipTestMethodUnderscores") + .desc("Skip underscores in test methods") + .defaultValue(false) + .build(); + public MethodNamingConventionsRule() { + definePropertyDescriptor(SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR); + setProperty(CODECLIMATE_CATEGORIES, "Style"); // Note: x10 as Apex has not automatic refactoring setProperty(CODECLIMATE_REMEDIATION_MULTIPLIER, 1); @@ -30,6 +41,9 @@ public class MethodNamingConventionsRule extends AbstractApexRule { if (isOverriddenMethod(node) || isPropertyAccessor(node) || isConstructor(node)) { return data; } + if (isTestMethod(node) && getProperty(SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR)) { + return data; + } String methodName = node.getImage(); @@ -53,4 +67,9 @@ public class MethodNamingConventionsRule extends AbstractApexRule { private boolean isConstructor(ASTMethod node) { return node.getNode().getMethodInfo().isConstructor(); } + + private boolean isTestMethod(ASTMethod node) { + final ASTModifierNode modifierNode = node.getFirstChildOfType(ASTModifierNode.class); + return modifierNode != null && modifierNode.getNode().getModifiers().isTest(); + } } diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml index e5229bcc42..67277abef6 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml @@ -66,4 +66,16 @@ public class Foo { ]]> + + #1573 method names should not contain underscores, but skip test methods + 0 + true + + + From 86c72a87b43c8464e15432ef68005629b70abc4c Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 4 Feb 2019 15:29:23 +0530 Subject: [PATCH 39/90] Try Statement added to list of block parents ASTTryStatement added to BLOCK_PARENTS. --- .../ConsecutiveLiteralAppendsRule.java | 2 ++ .../xml/ConsecutiveLiteralAppends.xml | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java index 17378c54bf..c79a11b619 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java @@ -26,6 +26,7 @@ import net.sourceforge.pmd.lang.java.ast.ASTPrimaryExpression; import net.sourceforge.pmd.lang.java.ast.ASTPrimarySuffix; import net.sourceforge.pmd.lang.java.ast.ASTSwitchLabel; import net.sourceforge.pmd.lang.java.ast.ASTSwitchStatement; +import net.sourceforge.pmd.lang.java.ast.ASTTryStatement; import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclaratorId; import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer; import net.sourceforge.pmd.lang.java.ast.ASTWhileStatement; @@ -74,6 +75,7 @@ public class ConsecutiveLiteralAppendsRule extends AbstractJavaRule { BLOCK_PARENTS.add(ASTIfStatement.class); BLOCK_PARENTS.add(ASTSwitchStatement.class); BLOCK_PARENTS.add(ASTMethodDeclaration.class); + BLOCK_PARENTS.add(ASTTryStatement.class); } private static final PropertyDescriptor THRESHOLD_DESCRIPTOR diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml index 9e3d41b59f..2e6f2743e5 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml @@ -1415,6 +1415,30 @@ public class ConsecutiveLiteralAppendsFP { } System.out.println(sb.toString()); } +} + ]]> + + + + #1632 [java] ConsecutiveLiteralAppends false positive + 0 + From 7dd44ec4140800cae302a7883d0ddb8836848a1e Mon Sep 17 00:00:00 2001 From: tomdaly Date: Mon, 4 Feb 2019 10:54:51 +0000 Subject: [PATCH 40/90] Fix PR comments --- .../codestyle/xml/MethodNamingConventions.xml | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml index 67277abef6..fcda99cba6 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/codestyle/xml/MethodNamingConventions.xml @@ -67,15 +67,37 @@ public class Foo { - #1573 method names should not contain underscores, but skip test methods + #1573 method names should not contain underscores, but skip test methods 1 0 true + + #1573 method names should not contain underscores, but skip test methods 2 + 0 + true + + + + + #1573 method names should not contain underscores, but skip test methods 3 + 1 + false + + From 709f9ed4ea4e9bc58a800251671a00c9ab8ee955 Mon Sep 17 00:00:00 2001 From: Shubham Date: Tue, 5 Feb 2019 12:58:39 +0530 Subject: [PATCH 41/90] Catch and Finally Statements added --- .../performance/ConsecutiveLiteralAppendsRule.java | 4 ++++ .../performance/xml/ConsecutiveLiteralAppends.xml | 13 ++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java index c79a11b619..90fa651331 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java @@ -16,7 +16,9 @@ import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.java.ast.ASTAdditiveExpression; import net.sourceforge.pmd.lang.java.ast.ASTAllocationExpression; import net.sourceforge.pmd.lang.java.ast.ASTArgumentList; +import net.sourceforge.pmd.lang.java.ast.ASTCatchStatement; import net.sourceforge.pmd.lang.java.ast.ASTDoStatement; +import net.sourceforge.pmd.lang.java.ast.ASTFinallyStatement; import net.sourceforge.pmd.lang.java.ast.ASTForStatement; import net.sourceforge.pmd.lang.java.ast.ASTIfStatement; import net.sourceforge.pmd.lang.java.ast.ASTLiteral; @@ -76,6 +78,8 @@ public class ConsecutiveLiteralAppendsRule extends AbstractJavaRule { BLOCK_PARENTS.add(ASTSwitchStatement.class); BLOCK_PARENTS.add(ASTMethodDeclaration.class); BLOCK_PARENTS.add(ASTTryStatement.class); + BLOCK_PARENTS.add(ASTCatchStatement.class); + BLOCK_PARENTS.add(ASTFinallyStatement.class); } private static final PropertyDescriptor THRESHOLD_DESCRIPTOR diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml index 2e6f2743e5..63922cee8d 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml @@ -1429,9 +1429,16 @@ public final class Test { sb.append("literal1"); try { final String res = bar(); - sb.append("literal2").append(res); - } catch (Exception e) { + sb.append(res); + sb.append("literal2"); + } catch (ArithmeticException e) { sb.append("literal3"); + } catch (ArrayIndexOutOfBoundException e) { + sb.append("literal4"); + } catch (Exception e) { + sb.append("literal5"); + } finally { + sb.append("literal6"); } return sb.toString(); } @@ -1439,7 +1446,7 @@ public final class Test { public String bar() throws Exception { throw new Exception(); } -} +} ]]> From 58b52a4fd8ce9a63b94ebf5e933817c940d0245e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Tue, 5 Feb 2019 22:49:53 -0300 Subject: [PATCH 42/90] Add more test cases --- .../xml/ConsecutiveLiteralAppends.xml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml index 63922cee8d..76595d13f4 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/performance/xml/ConsecutiveLiteralAppends.xml @@ -1446,6 +1446,46 @@ public final class Test { public String bar() throws Exception { throw new Exception(); } +} + ]]> + + + + Consecutive literal append over try + 1 + 4 + + + + + literal appends over lambdas + 0 + sb.append("foo"); + Runnable r2 = () -> sb.append("bar"); + + r.run(); + System.out.println(sb.toString()); + r2.run(); + return sb.toString(); + } } ]]> From 2165fb056cbb0c5d312e1a46e45823ca3a4dfaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Tue, 5 Feb 2019 22:50:11 -0300 Subject: [PATCH 43/90] Remove TryStatement from parent blocks - We should treat appends over the try edge as consecutive, it's only catch / finally that maye break the linear execution flow --- .../java/rule/performance/ConsecutiveLiteralAppendsRule.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java index 90fa651331..67644baf83 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/performance/ConsecutiveLiteralAppendsRule.java @@ -28,7 +28,6 @@ import net.sourceforge.pmd.lang.java.ast.ASTPrimaryExpression; import net.sourceforge.pmd.lang.java.ast.ASTPrimarySuffix; import net.sourceforge.pmd.lang.java.ast.ASTSwitchLabel; import net.sourceforge.pmd.lang.java.ast.ASTSwitchStatement; -import net.sourceforge.pmd.lang.java.ast.ASTTryStatement; import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclaratorId; import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer; import net.sourceforge.pmd.lang.java.ast.ASTWhileStatement; @@ -77,7 +76,6 @@ public class ConsecutiveLiteralAppendsRule extends AbstractJavaRule { BLOCK_PARENTS.add(ASTIfStatement.class); BLOCK_PARENTS.add(ASTSwitchStatement.class); BLOCK_PARENTS.add(ASTMethodDeclaration.class); - BLOCK_PARENTS.add(ASTTryStatement.class); BLOCK_PARENTS.add(ASTCatchStatement.class); BLOCK_PARENTS.add(ASTFinallyStatement.class); } From 3f436ddbbda5a66aa6aae72ee318abc4f2f7b5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Tue, 5 Feb 2019 22:52:54 -0300 Subject: [PATCH 44/90] Update changelog, refs #1645 --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index e9e0362965..a3adb6804e 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -28,6 +28,8 @@ This is a {{ site.pmd.release_type }} release. * [#780](https://github.com/pmd/pmd/issues/780): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors * java-multithreading * [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat +* java-performance + * [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch ### API Changes @@ -40,6 +42,7 @@ This is a {{ site.pmd.release_type }} release. * [#1635](https://github.com/pmd/pmd/pull/1635): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat - [Shubham](https://github.com/Shubham-2k17) * [#1637](https://github.com/pmd/pmd/pull/1637): \[java] Compile time constants initialized by literals avoided by AccessorMethodGenerationRule - [Shubham](https://github.com/Shubham-2k17) * [#1640](https://github.com/pmd/pmd/pull/1640): \[java] Update instead of override classHasLombokAnnotation flag - [Phokham Nonava](https://github.com/fluxroot) +* [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) {% endtocmaker %} From be73856c7c7974d00388f9728beeefd35370aaf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Tue, 5 Feb 2019 23:32:52 -0300 Subject: [PATCH 45/90] Update changelog, refs #1644 --- docs/pages/release_notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index e9e0362965..d2976c7d87 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,12 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### Modified Rules + +* The Apex rule {% rule "apex/codestyle/MethodNamingConventions" %} (apex-codestyle) has a new + property `skipTestMethodUnderscores`, which is by default disabled. The new property allows for ignoring + all test methods, either using the `testMethod` modifier or simply annotating them `@isTest`. + ### Fixed Issues * java-bestpractices @@ -40,6 +46,7 @@ This is a {{ site.pmd.release_type }} release. * [#1635](https://github.com/pmd/pmd/pull/1635): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat - [Shubham](https://github.com/Shubham-2k17) * [#1637](https://github.com/pmd/pmd/pull/1637): \[java] Compile time constants initialized by literals avoided by AccessorMethodGenerationRule - [Shubham](https://github.com/Shubham-2k17) * [#1640](https://github.com/pmd/pmd/pull/1640): \[java] Update instead of override classHasLombokAnnotation flag - [Phokham Nonava](https://github.com/fluxroot) +* [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly) {% endtocmaker %} From 1d321d9a0a68e6e1b3cff1e1a74b4efc8cd06842 Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 6 Feb 2019 16:51:46 +0530 Subject: [PATCH 46/90] Removed var from getting checked by UseDiamondOperatorRule Also includes Minor Correction in one test case. --- .../resources/category/java/codestyle.xml | 1 + .../rule/codestyle/xml/UseDiamondOperator.xml | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pmd-java/src/main/resources/category/java/codestyle.xml b/pmd-java/src/main/resources/category/java/codestyle.xml index 9066684bed..86025bc1ea 100644 --- a/pmd-java/src/main/resources/category/java/codestyle.xml +++ b/pmd-java/src/main/resources/category/java/codestyle.xml @@ -1952,6 +1952,7 @@ which makes the code also more readable. Use Diamond 2 - 8,11 + 6,8 + + #1624[java] UseDiamondOperator doesn't work with var + 1 + 6 + (); + f = new ArrayList<>(); + f = new ArrayList(); // flagged by rule + var s = new ArrayList<>(4); // won't compile + var b; // won't compile + } +} + ]]> + \ No newline at end of file From 50f64f5fb13c985d1cd015132933f52a5769f27d Mon Sep 17 00:00:00 2001 From: Shubham Date: Thu, 7 Feb 2019 10:46:06 +0530 Subject: [PATCH 47/90] Update UseDiamondOperator.xml --- .../pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml index 079fc4d0bd..de75597190 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml @@ -70,8 +70,6 @@ public class Buzz { var f = new ArrayList(); f = new ArrayList<>(); f = new ArrayList(); // flagged by rule - var s = new ArrayList<>(4); // won't compile - var b; // won't compile } } ]]> From 5bb24d96cfd0ba50919f94b48fa47c4907a911ac Mon Sep 17 00:00:00 2001 From: Shubham Date: Fri, 8 Feb 2019 17:26:35 +0530 Subject: [PATCH 48/90] XPATH for UseDiamondOperator Rule Updated --- pmd-java/src/main/resources/category/java/codestyle.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pmd-java/src/main/resources/category/java/codestyle.xml b/pmd-java/src/main/resources/category/java/codestyle.xml index 86025bc1ea..d2a60ffa5b 100644 --- a/pmd-java/src/main/resources/category/java/codestyle.xml +++ b/pmd-java/src/main/resources/category/java/codestyle.xml @@ -1951,8 +1951,7 @@ which makes the code also more readable. Date: Sat, 9 Feb 2019 16:31:09 +0530 Subject: [PATCH 49/90] New test case added --- .../pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml index de75597190..b78f4f06cd 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml @@ -61,8 +61,8 @@ public class Foo { #1624[java] UseDiamondOperator doesn't work with var - 1 - 6 + 2 + 6,7 (); f = new ArrayList<>(); f = new ArrayList(); // flagged by rule + List foo, bar = new ArrayList(), baz = new ArrayList<>(); //flagged only once } } ]]> From 189c5990f43288a254ffa2640b650379adecff89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sat, 9 Feb 2019 13:10:41 -0300 Subject: [PATCH 50/90] Split test cases for granularity --- docs/pages/pmd/rules/java/codestyle.md | 2 +- .../rule/codestyle/xml/UseDiamondOperator.xml | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/pages/pmd/rules/java/codestyle.md b/docs/pages/pmd/rules/java/codestyle.md index 9ab0b79baa..2124e2cd3c 100644 --- a/docs/pages/pmd/rules/java/codestyle.md +++ b/docs/pages/pmd/rules/java/codestyle.md @@ -2110,7 +2110,7 @@ which makes the code also more readable. **This rule is defined by the following XPath expression:** ``` xpath -//VariableInitializer +//VariableInitializer[preceding-sibling::VariableDeclaratorId[1]/@TypeInferred="false"] //PrimaryExpression[not(PrimarySuffix)] [not(ancestor::ArgumentList)] /PrimaryPrefix/AllocationExpression[ClassOrInterfaceType[@AnonymousClass='false']/TypeArguments//ReferenceType[not(.//TypeArguments)]] diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml index b78f4f06cd..7d61c4a29a 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UseDiamondOperator.xml @@ -61,18 +61,33 @@ public class Foo { #1624[java] UseDiamondOperator doesn't work with var - 2 - 6,7 + 1 + 6 (); - f = new ArrayList<>(); - f = new ArrayList(); // flagged by rule - List foo, bar = new ArrayList(), baz = new ArrayList<>(); //flagged only once + var f = new ArrayList(); // ok + f = new ArrayList<>(); // ok + f = new ArrayList(); // flagged by rule } } ]]> - \ No newline at end of file + + Multiple initializations in a single declaration + 1 + 6 + foo, + bar = new ArrayList(), // flagged + baz = new ArrayList<>(); // ok + } +} + ]]> + + From f012dca515431242bb3d45ed52e5958e47ab279f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sat, 9 Feb 2019 13:12:47 -0300 Subject: [PATCH 51/90] Update changelog, refs #1646 --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 1d2c22199b..ee410dcd04 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -28,6 +28,7 @@ This is a {{ site.pmd.release_type }} release. * java-codestyle * [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods * [#1547](https://github.com/pmd/pmd/issues/1547): \[java] AtLeastOneConstructorRule: false-positive with lombok.AllArgsConstructor + * [#1624](https://github.com/pmd/pmd/issues/1624): \[java] UseDiamondOperator false positive with var initializer * java-design * [#1641](https://github.com/pmd/pmd/issues/1641): \[java] False-positive with Lombok and inner classes * java-errorprone @@ -50,6 +51,7 @@ This is a {{ site.pmd.release_type }} release. * [#1640](https://github.com/pmd/pmd/pull/1640): \[java] Update instead of override classHasLombokAnnotation flag - [Phokham Nonava](https://github.com/fluxroot) * [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly) * [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) +* [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17) {% endtocmaker %} From 544238244beab6e6701df3675f362b3fb079582d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 10 Feb 2019 09:42:41 +0100 Subject: [PATCH 52/90] [java] Rename UseDiamondOperator -> UseDiamondOperatorTest --- .../{UseDiamondOperator.java => UseDiamondOperatorTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/{UseDiamondOperator.java => UseDiamondOperatorTest.java} (80%) diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperator.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperatorTest.java similarity index 80% rename from pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperator.java rename to pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperatorTest.java index 31c663fc17..b502195246 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperator.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/UseDiamondOperatorTest.java @@ -6,6 +6,6 @@ package net.sourceforge.pmd.lang.java.rule.codestyle; import net.sourceforge.pmd.testframework.PmdRuleTst; -public class UseDiamondOperator extends PmdRuleTst { +public class UseDiamondOperatorTest extends PmdRuleTst { // no additional unit tests } From fcba29a3b733124119c4672fdd50df9e61b92c73 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 25 Jan 2019 15:34:06 +0100 Subject: [PATCH 53/90] [plsql] ParseException when using hierarchical query clause Fixes #1590 * Moves function calls down to primary expression so that functions can be used in a select expression * Restrict FunctionCall to built-in functions. User defined functions are parsed as primary expressions. * Parse function name of built-in function as general ID. * The Tokens _DEFAULT, ELSE and EXCEPTION are reserved words and not available for identifiers. --- pmd-plsql/etc/grammar/PldocAST.jjt | 177 +++++++++--------- .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 + .../lang/plsql/rule/AbstractPLSQLRule.java | 5 + .../plsql/rule/codestyle/CodeFormatRule.java | 8 +- .../lang/plsql/ast/SelectExpressionsTest.java | 2 - .../plsql/ast/SelectHierarchicalTest.java | 24 +++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 8 +- .../pmd/lang/plsql/ast/SelectExpressions.pls | 4 + .../pmd/lang/plsql/ast/SelectHierarchical.pls | 90 +++++++++ 9 files changed, 227 insertions(+), 96 deletions(-) create mode 100644 pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchicalTest.java create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchical.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index f08f27c0cc..91d39a0938 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -357,7 +357,7 @@ ASTPackageSpecification PackageSpecification() : } { ( - [ [ ] [ | ] ] + [ [ "REPLACE"] [ | ] ] simpleNode = ObjectNameDeclaration() ( @@ -388,7 +388,7 @@ ASTPackageBody PackageBody() : } { ( - [ [ ] [ | ] ] + [ [ "REPLACE"] [ | ] ] ( | ) simpleNode = ObjectNameDeclaration() ( @@ -490,7 +490,7 @@ ASTProgramUnit ProgramUnit() : { ( - [ [ ] [ | ] ] + [ [ "REPLACE"] [ | ] ] MethodDeclarator() @@ -684,7 +684,7 @@ ASTDatatype Datatype() : LOOKAHEAD(2) simpleNode = ScalarDataTypeName() {sb.append(simpleNode.getImage());} | ( - ( [LOOKAHEAD(2) {sb.append(token.image);} ] simpleNode = QualifiedName() {sb.append(simpleNode.getImage());} + ( [LOOKAHEAD(2) "REF" {sb.append(token.image);} ] simpleNode = QualifiedName() {sb.append(simpleNode.getImage());} //Bug 35352414 - datatype may include dblink ["@" simpleNode = QualifiedName() {sb.append("@"+simpleNode.getImage());} ] ["%" (|){sb.append("%"+token.image);} ] @@ -796,7 +796,7 @@ ASTScalarDataTypeName ScalarDataTypeName() : // reference types | //SRT Added to support pre-defined weak REF CURSOR - ( {name.append("REF CURSOR");}) | + ("REF" {name.append("REF CURSOR");}) | // object_type - defined elsewhere // scalar types - date/time: @@ -1145,6 +1145,7 @@ void RestOfStatement() #void : { FromClause() [ WhereClause() ] + [ HierarchicalQueryClause() ] [ GroupByClause() ] [ OrderByClause() ] [ RowLimitingClause() ] @@ -1190,7 +1191,7 @@ void OrderByEntry() #void : { ( LOOKAHEAD(2) ColumnAlias() | LOOKAHEAD(2) SqlExpression() ) [ | ] - [ LOOKAHEAD(2) | LOOKAHEAD(2) ] + [ LOOKAHEAD(2) "FIRST" | LOOKAHEAD(2) "LAST" ] } /** @@ -1202,7 +1203,7 @@ ASTRowLimitingClause RowLimitingClause() : ( NumericLiteral() ( | ) | - ( | ) [ NumericLiteral() [ ] ] ( | ) ( | ) + ( "FIRST" | ) [ NumericLiteral() [ ] ] ( | ) ( | ) ) { return jjtThis; } } @@ -1215,7 +1216,7 @@ ASTQueryBlock QueryBlock() : SelectList() FromClause() [ WhereClause() ] - // [ HierarchicalQueryClause() ] + [ HierarchicalQueryClause() ] [ GroupByClause() ] // [ ModelClause() ] { return jjtThis; } @@ -1403,9 +1404,10 @@ ASTExpressionList ExpressionList() : ASTSqlExpression SqlExpression() : {} { + [ LOOKAHEAD(2) ] // SimpleExpression ( - LOOKAHEAD(FunctionCall()) FunctionCall() + LOOKAHEAD(3) AdditiveExpression() // this can be a literal or a simple expression, but no conditional | LOOKAHEAD(5) SchemaName() "." TableName() "." Column() | @@ -1414,37 +1416,24 @@ ASTSqlExpression SqlExpression() : LOOKAHEAD(2) Column() | LOOKAHEAD(2) - | - AdditiveExpression() // this can be a literal or a simple expression, but no conditional ) { return jjtThis; } } /** - * See also https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/About-User-Defined-Functions.html#GUID-4EB3E236-8216-471C-BA44-23D87BDFEA67 - * - * A function reference might be: - * function_name - * package.function_name - * package.schema.function_name - * optional: @ dblink . + * Built-in function call + * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Functions.html#GUID-D079EFD3-C683-441F-977E-2C9503089982 */ ASTFunctionCall FunctionCall() : { ASTID id; - StringBuilder name = new StringBuilder(); } { - id = ID() { name.append(id.getImage()); } - [ "." id = ID() { name.append('.').append(id.getImage()); } - [ "." id = ID() { name.append('.').append(id.getImage()); } ] - ] - [ "@" id = ID() { name.append('@').append(id.getImage()); } ] - + id = ID() Arguments() { - jjtThis.setImage(name.toString()); + jjtThis.setImage(id.getImage()); return jjtThis; } } @@ -1456,6 +1445,20 @@ ASTColumn Column() : { return jjtThis; } } +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/SELECT.html#GUID-CFA006CA-6FF1-4972-821E-6996142A51C6__I2126079 + */ +ASTHierarchicalQueryClause HierarchicalQueryClause() : +{} +{ + ( + ( [ LOOKAHEAD(2) ] Condition() [ Condition() ] ) + | + ( Condition() [ LOOKAHEAD(2) ] Condition() ) + ) + { return jjtThis; } +} + ASTFromClause FromClause() : {} { @@ -1497,19 +1500,14 @@ ASTSelectList SelectList() : void SelectListEntry() #void : {} { - ( - LOOKAHEAD(3) TableAlias() "." "*" - | - LOOKAHEAD(FunctionCall()) FunctionCall() [ [] ColumnAlias() ] - | - LOOKAHEAD(3) Expression() [ [] ColumnAlias() ] - ) + [ LOOKAHEAD(2) ] + SqlExpression() [LOOKAHEAD(2, {!(getToken(1).kind == BULK)}) [LOOKAHEAD(2) ] ColumnAlias() ] } ASTColumnAlias ColumnAlias() : -{} +{ ASTID id; } { - ( | ) { jjtThis.setImage(token.image); } + id = ID() {jjtThis.setImage(id.getImage());} { return jjtThis; } } @@ -1957,7 +1955,7 @@ ASTMultiTableInsert MultiTableInsert() : ASTConditionalInsertClause ConditionalInsertClause() : {} { - [ | ] ( Condition() ( InsertIntoClause() [ ValuesClause() ] )+ )+ + [ | "FIRST" ] ( Condition() ( InsertIntoClause() [ ValuesClause() ] )+ )+ [ ( InsertIntoClause() [ ValuesClause() ] )+ ] { return jjtThis; } } @@ -2231,7 +2229,7 @@ ASTSubTypeDefinition SubTypeDefinition() : (( | | )["(" NumericLiteral() ")"] Datatype() ( )? ( Datatype())?) | - [ Datatype()] + "REF" [ Datatype()] //Enumeration | ( "(" Expression() @@ -2511,7 +2509,7 @@ ASTTrimExpression TrimExpression() : { Token thisToken; PLSQLNode simpleNode = null; StringBuilder sb = new StringBuilder() ; } { ( - (thisToken = ) { sb.append(thisToken.image);} + (thisToken = "TRIM" ) { sb.append(thisToken.image);} "(" { sb.append("(");} [ ( | | ){ sb.append(" "); sb.append(token.toString()); } ] [ simpleNode = StringExpression() { sb.append(" "); sb.append(simpleNode.getImage()); } ] @@ -2799,6 +2797,14 @@ ASTIsOfTypeCondition IsOfTypeCondition() #IsOfTypeCondition(>1) : * 2006-05-23 - Matthias Hendler - Added lookahead otherwise warning encountered. * Warning arised while adding methode triggerUnit(). * 2011-04-27 - SRT - Add optional NEW Keyword to cope with Object Type constructors + * + * See also https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/About-User-Defined-Functions.html#GUID-4EB3E236-8216-471C-BA44-23D87BDFEA67 + * + * A function reference might be: + * function_name + * package.function_name + * package.schema.function_name + * optional: @ dblink . */ ASTPrimaryExpression PrimaryExpression() #PrimaryExpression(>1) : { Token thisToken ; PLSQLNode simpleNode = null; StringBuilder sb = new StringBuilder() ; @@ -2810,7 +2816,7 @@ ASTPrimaryExpression PrimaryExpression() #PrimaryExpression(>1) : ) | ( - LOOKAHEAD( PrimaryPrefix() ) // Lookahead so that we can recover and treat NEW as an identifier + LOOKAHEAD( PrimaryPrefix() ) // Lookahead so that we can recover and treat NEW as an identifier { sb.append(" NEW "); } (simpleNode = PrimaryPrefix() ) {sb.append(simpleNode.getImage());} | (simpleNode = PrimaryPrefix() ) {sb.append(simpleNode.getImage());} @@ -2829,7 +2835,9 @@ ASTPrimaryPrefix PrimaryPrefix() : } { ( - ( simpleNode = Literal() ) { sb.append(simpleNode.getImage()) ; } + LOOKAHEAD(2) ( + LOOKAHEAD(2) simpleNode = FunctionCall() +| LOOKAHEAD(2) simpleNode = Literal() ) { sb.append(simpleNode.getImage()) ; } | LOOKAHEAD(MultiSetCondition()) simpleNode = MultiSetCondition() | LOOKAHEAD(TrimExpression()) simpleNode = TrimExpression() //SRT 20110613.3 | LOOKAHEAD(CaseExpression()) ( simpleNode =CaseExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110520 @@ -3285,7 +3293,7 @@ ASTView View() : PLSQLNode simpleNode = null; } { - [ ] + [ "REPLACE"] [[] ] simpleNode = ObjectNameDeclaration() ["(" ViewColumn() ("," ViewColumn())* ")"] @@ -3302,7 +3310,7 @@ ASTSynonym Synonym() : PLSQLNode simpleNode = null; } { - [ ] + [ "REPLACE"] [] simpleNode = ObjectNameDeclaration() @@ -3317,7 +3325,7 @@ ASTDirectory Directory() : PLSQLNode simpleNode = null; } { - [ ] + [ "REPLACE"] simpleNode = ObjectNameDeclaration() @@ -3434,7 +3442,7 @@ ASTTypeSpecification TypeSpecification() : PLSQLNode simpleNode = null; } { - [ [ ] [ | ] ] + [ [ "REPLACE"] [ | ] ] simpleNode = ObjectNameDeclaration() [ @@ -3613,7 +3621,7 @@ ASTAlterTypeSpec AlterTypeSpec() : | */ [ - + "REPLACE" ( LOOKAHEAD(2) // ( | ) ( @@ -3825,7 +3833,7 @@ ASTTriggerUnit TriggerUnit() : PLSQLNode simpleNode = null ; } { - [ [ ] [ | ] ] + [ [ "REPLACE"] [ | ] ] () simpleNode = ObjectNameDeclaration() @@ -4071,7 +4079,6 @@ MORE : TOKEN [IGNORE_CASE]: { - | | | | @@ -4082,7 +4089,10 @@ TOKEN [IGNORE_CASE]: | // PRAGMA INLINE } -/* PL/SQL RESERVED WORDS */ +/** + * PL/SQL RESERVED WORDS + * https://docs.oracle.com/en/database/oracle/oracle-database/18/lnpls/plsql-reserved-words-keywords.html#GUID-9BAA3A99-41B1-45CB-A91E-1E482BC1F927 + */ /** * 2006-05-20 - Matthias Hendler - Removed: * Added: , , , @@ -4106,7 +4116,6 @@ TOKEN [IGNORE_CASE]: | | | - | | | | @@ -4177,7 +4186,6 @@ TOKEN [IGNORE_CASE]: | | | - | | | | @@ -4221,7 +4229,6 @@ TOKEN [IGNORE_CASE]: | | // | - | | | | @@ -4233,11 +4240,9 @@ TOKEN [IGNORE_CASE]: | | | - | | | | - | | | | @@ -4311,7 +4316,6 @@ TOKEN [IGNORE_CASE]: | | | - | | | | @@ -4351,11 +4355,9 @@ TOKEN [IGNORE_CASE]: | | | - | | | | - | | | | @@ -4489,7 +4491,6 @@ TOKEN [IGNORE_CASE]: | | | -| | | | @@ -4510,9 +4511,10 @@ TOKEN [IGNORE_CASE]: | | | - | | +| +| } /** @@ -4596,7 +4598,11 @@ TOKEN : } -//SRT 2011-04-17 - START +/** + * PL/SQL Reserved words + * + * https://docs.oracle.com/en/database/oracle/oracle-database/18/lnpls/plsql-reserved-words-keywords.html#GUID-9BAA3A99-41B1-45CB-A91E-1E482BC1F927 + */ ASTKEYWORD_RESERVED KEYWORD_RESERVED (): {} { // PL/SQL RESERVED WORDS - V$RESERVED.RESERVED='Y' @@ -4687,7 +4693,9 @@ ASTKEYWORD_RESERVED KEYWORD_RESERVED (): {} ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} { // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' -( +( +"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | + | | //| @@ -4824,7 +4832,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| +| //| //| //| @@ -4889,7 +4897,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| //| //| //| @@ -4977,7 +4984,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -| //| //| //| @@ -5098,7 +5104,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| | -| | //| //| @@ -5146,7 +5151,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -| +//| //| //| //| @@ -5169,7 +5174,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -| +//| //| //| //| @@ -5289,7 +5294,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -//| +| //| //| //| @@ -5464,7 +5469,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -| //| //| //| @@ -5481,7 +5485,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -//| //| //| //| @@ -5674,7 +5677,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| | //| -| | //| //| @@ -5732,7 +5734,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -//| | | | @@ -5798,7 +5799,6 @@ ASTID ID(): {} | | | | | | | | | | | | | */ - | //SYNTAX //20120501 | | | | | // | @@ -5810,7 +5810,7 @@ ASTID ID(): {} | //SYNTAX //RESERVED WORD | //RESERVED WORD //20120429 | | - | + //| // | | //RESERVED WORD | @@ -5827,7 +5827,7 @@ ASTID ID(): {} | //- | //20120501 | | //RESERVED WPRDS - | //SYNTAX //RESERVED WORD + //| //SYNTAX //RESERVED WORD | | //SYNTAX //RESERVED WORD //20120501 | @@ -5835,17 +5835,17 @@ ASTID ID(): {} | //RESERVED WORD | //SYNTAX | //RESERVED WORD - | <_DEFAULT> //RESERVED WORD + //| <_DEFAULT> //RESERVED WORD | //RESERVED WORD | //RESERVED WORD //| //RESERVED WORD | | //RESERVED WORD - | //SYNTAX //RESERVED WORD + //| //SYNTAX //RESERVED WORD | //SYNTAX //| | - | //SYNTAX + //| //SYNTAX | //SYNTAX //RESERVED WORD | //SYNTAX | //SYNTAX //RESERVED WORD @@ -5855,7 +5855,7 @@ ASTID ID(): {} | //SYNTAX //RESERVED WORD | //RESERVED WORD | //SYNTAX - | //RESERVED WORD + //| //RESERVED WORD // | | // this causes bug 643043 Procedure w/o params appears as variable | //SYNTAX @@ -5872,7 +5872,7 @@ ASTID ID(): {} | | //RESERVED WORD //20120501 | - | //RESERVED WORD + //| //RESERVED WORD | //SYNTAX //20120501 | | | | //RESERVED WORD @@ -5921,7 +5921,7 @@ ASTID ID(): {} | //SYNTAX | //RESERVED WORD //20120501 | - // | | | | + // | | | //| //20120501 | //20120501 | //SYNTAX @@ -5935,10 +5935,10 @@ ASTID ID(): {} | //RESERVED WORD | | | - | //RESERVED WORD - | // | + //| //RESERVED WORD + //| // | //20120501 | - | + //| | //RESERVED WORD | //|
//RESERVED WORD @@ -6002,7 +6002,6 @@ ASTUnqualifiedID UnqualifiedID(): {} | | | - | | | | @@ -6041,7 +6040,6 @@ ASTQualifiedID QualifiedID(): {} | --Unreserved Key Word | //SRT */ - | //20120501 | //| | @@ -6058,7 +6056,7 @@ ASTQualifiedID QualifiedID(): {} // // //20120429 | - | + //| // // | @@ -6183,7 +6181,6 @@ ASTQualifiedID QualifiedID(): {} | //20120501 | //| - //| //| //| //20120501 | @@ -6203,10 +6200,10 @@ ASTQualifiedID QualifiedID(): {} | | // - | + //| //| //20120501 | - | + //| | | //
@@ -6273,7 +6270,7 @@ ASTTypeKeyword TypeKeyword(): {} | | | - | | + | "REF" | | | | | | diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 9d86b884b8..6cfacdb9b3 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -986,4 +986,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTValuesClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTHierarchicalQueryClause node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index f43818e62d..28a593edac 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1080,6 +1080,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTHierarchicalQueryClause node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/codestyle/CodeFormatRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/codestyle/CodeFormatRule.java index db5d6031c4..30bd52b09a 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/codestyle/CodeFormatRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/codestyle/CodeFormatRule.java @@ -125,10 +125,14 @@ public class CodeFormatRule extends AbstractPLSQLRule { int currentLine = firstLine; for (int i = 0; i < parent.jjtGetNumChildren(); i++) { Node child = parent.jjtGetChild(i); + String image = child.getImage(); + if (image == null && child.jjtGetNumChildren() > 0) { + image = child.jjtGetChild(0).getImage(); + } if (child.getBeginLine() != currentLine) { - addViolationWithMessage(data, child, child.getImage() + " should be on line " + currentLine); + addViolationWithMessage(data, child, image + " should be on line " + currentLine); } else if (i > 0 && child.getBeginColumn() != indentation) { - addViolationWithMessage(data, child, child.getImage() + " should begin at column " + indentation); + addViolationWithMessage(data, child, image + " should begin at column " + indentation); } // next entry needs to be on the next line currentLine++; diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java index b6264dfe94..a988f0b9bd 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java @@ -8,7 +8,6 @@ import java.nio.charset.StandardCharsets; import org.apache.commons.io.IOUtils; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import net.sourceforge.pmd.lang.plsql.AbstractPLSQLParserTst; @@ -16,7 +15,6 @@ import net.sourceforge.pmd.lang.plsql.AbstractPLSQLParserTst; public class SelectExpressionsTest extends AbstractPLSQLParserTst { @Test - @Ignore public void parseSelectExpression() throws Exception { String code = IOUtils.toString(this.getClass().getResourceAsStream("SelectExpressions.pls"), StandardCharsets.UTF_8); diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchicalTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchicalTest.java new file mode 100644 index 0000000000..f2809c982a --- /dev/null +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchicalTest.java @@ -0,0 +1,24 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.plsql.ast; + +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +import net.sourceforge.pmd.lang.plsql.AbstractPLSQLParserTst; + +public class SelectHierarchicalTest extends AbstractPLSQLParserTst { + + @Test + public void parseSelectHierarchicalQueries() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("SelectHierarchical.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + Assert.assertNotNull(input); + } +} diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 18d549472a..7edba49d3b 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -23,8 +23,12 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { List selectStatements = input.findDescendantsOfType(ASTSelectIntoStatement.class); Assert.assertEquals(3, selectStatements.size()); - ASTFunctionCall functionCall = selectStatements.get(2).getFirstDescendantOfType(ASTFunctionCall.class); - Assert.assertEquals("utils.get_colname", functionCall.getImage()); + ASTFunctionCall functionCall = selectStatements.get(0).getFirstDescendantOfType(ASTFunctionCall.class); + Assert.assertEquals("UPPER", functionCall.getImage()); + + ASTPrimaryPrefix primaryPrefix = selectStatements.get(2).getFirstDescendantOfType(ASTWhereClause.class) + .findDescendantsOfType(ASTPrimaryPrefix.class).get(1); + Assert.assertEquals("utils.get_colname", primaryPrefix.getImage()); } @Test diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index 06641367bd..a8084fd395 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -4,6 +4,9 @@ BEGIN +SELECT AVG(sal)*2 INTO foo FROM bar; + + SELECT AVG(salary) * 12 "Average Sal" INTO some_record @@ -14,5 +17,6 @@ SELECT INTO some_record FROM some_table; + END; / \ No newline at end of file diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchical.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchical.pls new file mode 100644 index 0000000000..84b7b20516 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectHierarchical.pls @@ -0,0 +1,90 @@ +-- +-- Select statement with hierarchical queries +-- +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/SELECT.html#GUID-CFA006CA-6FF1-4972-821E-6996142A51C6 +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/SELECT.html#GUID-CFA006CA-6FF1-4972-821E-6996142A51C6__I2130004 +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Hierarchical-Queries.html#GUID-0118DF1D-B9A9-41EB-8556-C6E7D6A5A84E +-- + +BEGIN + +SELECT id INTO v_id + FROM (SELECT separator_in || string_in || separator_in AS token_list FROM DUAL) + CONNECT BY col_length <= LENGTH(string_in) - LENGTH(separator_in); + +SELECT last_name, employee_id, manager_id + INTO test + FROM employees + CONNECT BY employee_id = manager_id + ORDER BY last_name; + +SELECT last_name, employee_id, manager_id + INTO test + FROM employees + CONNECT BY PRIOR employee_id = manager_id + AND salary > commission_pct + ORDER BY last_name; + +SELECT employee_id, last_name, manager_id + INTO test + FROM employees + CONNECT BY PRIOR employee_id = manager_id; + +SELECT employee_id, last_name, manager_id, LEVEL + INTO test + FROM employees + CONNECT BY PRIOR employee_id = manager_id; + +SELECT last_name, employee_id, manager_id, LEVEL + INTO test + FROM employees + START WITH employee_id = 100 + CONNECT BY PRIOR employee_id = manager_id + ORDER SIBLINGS BY last_name; + +SELECT last_name "Employee", + LEVEL, SYS_CONNECT_BY_PATH(last_name, '/') "Path" + INTO test + FROM employees + WHERE level <= 3 AND department_id = 80 + START WITH last_name = 'King' + CONNECT BY PRIOR employee_id = manager_id AND LEVEL <= 4; + +SELECT last_name "Employee", CONNECT_BY_ISCYCLE "Cycle", + LEVEL, SYS_CONNECT_BY_PATH(last_name, '/') "Path" + INTO test + FROM employees + WHERE level <= 3 AND department_id = 80 + START WITH last_name = 'King' + CONNECT BY NOCYCLE PRIOR employee_id = manager_id AND LEVEL <= 4 + ORDER BY "Employee", "Cycle", LEVEL, "Path"; + +SELECT LTRIM(SYS_CONNECT_BY_PATH (warehouse_id,','),',') + INTO test + FROM + (SELECT ROWNUM r, warehouse_id FROM warehouses) + WHERE CONNECT_BY_ISLEAF = 1 + START WITH r = 1 + CONNECT BY r = PRIOR r + 1 + ORDER BY warehouse_id; + +SELECT last_name "Employee", CONNECT_BY_ROOT last_name "Manager", + LEVEL-1 "Pathlen", SYS_CONNECT_BY_PATH(last_name, '/') "Path" + INTO test + FROM employees + WHERE LEVEL > 1 and department_id = 110 + CONNECT BY PRIOR employee_id = manager_id + ORDER BY "Employee", "Manager", "Pathlen", "Path"; + +SELECT name, SUM(salary) "Total_Salary" + INTO test + FROM ( + SELECT CONNECT_BY_ROOT last_name as name, Salary + FROM employees + WHERE department_id = 110 + CONNECT BY PRIOR employee_id = manager_id) + GROUP BY name + ORDER BY name, "Total_Salary"; + +END; +/ From adfc8511e64ac5c8d277d1812de65ed43d8732cd Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 9 Feb 2019 22:51:33 +0100 Subject: [PATCH 54/90] [plsql] FunctionCall is now used again for user defined functions --- pmd-plsql/etc/grammar/PldocAST.jjt | 46 +++++++++++++------ .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 ++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 ++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 7 ++- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 91d39a0938..2a308d2d6c 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1421,19 +1421,44 @@ ASTSqlExpression SqlExpression() : } /** - * Built-in function call + * Built-in function call or a user defined function call. * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Functions.html#GUID-D079EFD3-C683-441F-977E-2C9503089982 + * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/About-User-Defined-Functions.html#GUID-4EB3E236-8216-471C-BA44-23D87BDFEA67 + * + * A function reference/name might be: + * function_name + * package.function_name + * package.schema.function_name + * optional: @ dblink + * */ ASTFunctionCall FunctionCall() : { - ASTID id; + ASTFunctionName name; } { - id = ID() + name = FunctionName() Arguments() { - jjtThis.setImage(id.getImage()); + jjtThis.setImage(name.getImage()); + return jjtThis; + } +} + +ASTFunctionName FunctionName() : +{ + ASTID id; + StringBuilder name = new StringBuilder(); +} +{ + id = ID() { name.append(id.getImage()); } + [ "." id = ID() { name.append('.').append(id.getImage()); } + [ "." id = ID() { name.append('.').append(id.getImage()); } ] + ] + [ "@" id = ID() { name.append('@').append(id.getImage()); } ] + { + jjtThis.setImage(name.toString()); return jjtThis; } } @@ -2797,14 +2822,6 @@ ASTIsOfTypeCondition IsOfTypeCondition() #IsOfTypeCondition(>1) : * 2006-05-23 - Matthias Hendler - Added lookahead otherwise warning encountered. * Warning arised while adding methode triggerUnit(). * 2011-04-27 - SRT - Add optional NEW Keyword to cope with Object Type constructors - * - * See also https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/About-User-Defined-Functions.html#GUID-4EB3E236-8216-471C-BA44-23D87BDFEA67 - * - * A function reference might be: - * function_name - * package.function_name - * package.schema.function_name - * optional: @ dblink . */ ASTPrimaryExpression PrimaryExpression() #PrimaryExpression(>1) : { Token thisToken ; PLSQLNode simpleNode = null; StringBuilder sb = new StringBuilder() ; @@ -2835,9 +2852,8 @@ ASTPrimaryPrefix PrimaryPrefix() : } { ( - LOOKAHEAD(2) ( - LOOKAHEAD(2) simpleNode = FunctionCall() -| LOOKAHEAD(2) simpleNode = Literal() ) { sb.append(simpleNode.getImage()) ; } + LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() +| LOOKAHEAD(Literal()) simpleNode = Literal() { sb.append(simpleNode.getImage()) ; } | LOOKAHEAD(MultiSetCondition()) simpleNode = MultiSetCondition() | LOOKAHEAD(TrimExpression()) simpleNode = TrimExpression() //SRT 20110613.3 | LOOKAHEAD(CaseExpression()) ( simpleNode =CaseExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110520 diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 6cfacdb9b3..8d98a27d55 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -991,4 +991,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTHierarchicalQueryClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTFunctionName node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index 28a593edac..1161473e1b 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1085,6 +1085,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTFunctionName node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 7edba49d3b..717637f721 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -25,10 +25,9 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { ASTFunctionCall functionCall = selectStatements.get(0).getFirstDescendantOfType(ASTFunctionCall.class); Assert.assertEquals("UPPER", functionCall.getImage()); - - ASTPrimaryPrefix primaryPrefix = selectStatements.get(2).getFirstDescendantOfType(ASTWhereClause.class) - .findDescendantsOfType(ASTPrimaryPrefix.class).get(1); - Assert.assertEquals("utils.get_colname", primaryPrefix.getImage()); + + ASTFunctionCall functionCall2 = selectStatements.get(2).getFirstDescendantOfType(ASTFunctionCall.class); + Assert.assertEquals("utils.get_colname", functionCall2.getImage()); } @Test From c03910fb9950fbb848ebedd171987f9b7bc7f260 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 10 Feb 2019 09:17:42 +0100 Subject: [PATCH 55/90] Update release notes, refs #1590 --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index ee410dcd04..eba9922d5f 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -37,6 +37,8 @@ This is a {{ site.pmd.release_type }} release. * [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat * java-performance * [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch +* plsql + * [#1590](https://github.com/pmd/pmd/issues/1590): \[plsql] ParseException when using hierarchical query clause ### API Changes From fcdded6d5d5a92526d578457eaf549e6a2bb4352 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 23 Jan 2019 21:11:46 +0100 Subject: [PATCH 56/90] [plsql] ParseException with subqueries in WHERE clause Fixes #1589 --- docs/pages/release_notes.md | 2 ++ pmd-plsql/etc/grammar/PldocAST.jjt | 2 ++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 7 +++++++ .../pmd/lang/plsql/ast/WhereClauseSubqueries.pls | 14 ++++++++++++++ 4 files changed, 25 insertions(+) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index ee410dcd04..859fa26b03 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -37,6 +37,8 @@ This is a {{ site.pmd.release_type }} release. * [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat * java-performance * [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch +* plsql + * [#1589](https://github.com/pmd/pmd/issues/1589): \[plsql] ParseException with subqueries in WHERE clause ### API Changes diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index f08f27c0cc..017e786489 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1414,6 +1414,8 @@ ASTSqlExpression SqlExpression() : LOOKAHEAD(2) Column() | LOOKAHEAD(2) + | + LOOKAHEAD(2) "(" SelectStatement() ")" // see "Scalar Subquery Expressions" | AdditiveExpression() // this can be a literal or a simple expression, but no conditional ) diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 18d549472a..59141160fd 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -68,4 +68,11 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); } + + @Test + public void testSubqueries() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("WhereClauseSubqueries.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls new file mode 100644 index 0000000000..6c67d24e65 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls @@ -0,0 +1,14 @@ +-- +-- Where Clause with Subqueries +-- + +BEGIN + +SELECT id INTO v_id FROM table + WHERE id = (SELECT id FROM other_table); + +UPDATE table SET name = 'a' + WHERE id = (SELECT id FROM other_table); + +END; +/ From 06eb3e24ebef27b231cf6caa240cc5e235f649ad Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 24 Jan 2019 18:53:14 +0100 Subject: [PATCH 57/90] [plsql] Parse Exception with EXISTS Fixes #1587 --- docs/pages/release_notes.md | 2 ++ pmd-plsql/etc/grammar/PldocAST.jjt | 12 +++++++++- .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 ++++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 ++++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 7 ++++++ .../pmd/lang/plsql/ast/WhereClauseExists.pls | 24 +++++++++++++++++++ 6 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseExists.pls diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index ee410dcd04..51de4955c2 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -37,6 +37,8 @@ This is a {{ site.pmd.release_type }} release. * [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat * java-performance * [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch +* plsql + * [#1587](https://github.com/pmd/pmd/issues/1587): \[plsql] Parse Exception with EXISTS ### API Changes diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index f08f27c0cc..97d3b6e4c4 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1314,6 +1314,8 @@ void Condition2() #void : LOOKAHEAD(4) IsNullCondition() | LOOKAHEAD(4) IsOfTypeCondition() + | + ExistsCondition() ) } @@ -1338,6 +1340,14 @@ ASTLikeCondition LikeCondition() : { return jjtThis; } } +ASTExistsCondition ExistsCondition() : +{} +{ + "(" Subquery() ")" + + { return jjtThis; } +} + ASTCompoundCondition CompoundCondition() : {} { @@ -5848,7 +5858,7 @@ ASTID ID(): {} | //SYNTAX | //SYNTAX //RESERVED WORD | //SYNTAX - | //SYNTAX //RESERVED WORD + //| //SYNTAX //RESERVED WORD //| //SYNTAX //20120501 | | //SYNTAX diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 9d86b884b8..82887774e0 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -986,4 +986,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTValuesClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTExistsCondition node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index f43818e62d..2bbc0f5b03 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1080,6 +1080,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTExistsCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 18d549472a..7524bcea6f 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -68,4 +68,11 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); } + + @Test + public void testExistsCondition() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("WhereClauseExists.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseExists.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseExists.pls new file mode 100644 index 0000000000..e300dfb7a0 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseExists.pls @@ -0,0 +1,24 @@ +-- +-- Where Clause With Exists Condition +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/EXISTS-Condition.html#GUID-20259A83-C42B-4E0D-8DF4-9A2A66ACA8E7 +-- + +BEGIN + +SELECT id + INTO id_out + FROM some_table + WHERE EXISTS + (SELECT NULL + FROM other_table + WHERE other_id = other_id_in); + +DELETE FROM some_table + WHERE id = id_in + AND NOT EXISTS + (SELECT NULL + FROM other_table + WHERE id = id_in); + +END; +/ From 2e07139f8d3b8e7face847c87713259c49c45481 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 24 Jan 2019 19:23:13 +0100 Subject: [PATCH 58/90] [plsql] Add multiset condition in where clause --- pmd-plsql/etc/grammar/PldocAST.jjt | 44 ++++++++++++++++++- .../plsql/ast/PLSQLParserVisitorAdapter.java | 20 +++++++++ .../lang/plsql/rule/AbstractPLSQLRule.java | 20 +++++++++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 7 +++ .../lang/plsql/ast/WhereClauseMultiset.pls | 35 +++++++++++++++ 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseMultiset.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 97d3b6e4c4..c61770b02d 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1300,6 +1300,8 @@ void Condition2() #void : ( // a IS OF Type condition that starts with a function... LOOKAHEAD(FunctionCall() ) IsOfTypeCondition() + | + LOOKAHEAD(9) MultisetCondition() | LOOKAHEAD(9) ComparisonCondition() | @@ -1348,6 +1350,46 @@ ASTExistsCondition ExistsCondition() : { return jjtThis; } } +void MultisetCondition() #void : +{} +{ + LOOKAHEAD(4) IsASetCondition() + | + LOOKAHEAD(4) IsEmptyCondition() + | + LOOKAHEAD(4) MemberCondition() + | + LOOKAHEAD(4) SubmultisetCondition() +} + +ASTIsASetCondition IsASetCondition() : +{} +{ + TableName() [] + { return jjtThis; } +} + +ASTIsEmptyCondition IsEmptyCondition() : +{} +{ + TableName() [] + { return jjtThis; } +} + +ASTMemberCondition MemberCondition() : +{} +{ + SqlExpression() [] [] TableName() + { return jjtThis; } +} + +ASTSubmultisetCondition SubmultisetCondition() : +{} +{ + TableName() [] [] TableName() + { return jjtThis; } +} + ASTCompoundCondition CompoundCondition() : {} { @@ -5904,7 +5946,7 @@ ASTID ID(): {} | //RESERVED WORD | | - | //RESERVED WORD + //| //RESERVED WORD //| //RESERVED WORD //20120501 | | diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 82887774e0..770b089c2a 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -991,4 +991,24 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTExistsCondition node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTIsASetCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTIsEmptyCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTMemberCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTSubmultisetCondition node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index 2bbc0f5b03..66f58346c7 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1085,6 +1085,26 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTIsASetCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTIsEmptyCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTMemberCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTSubmultisetCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 7524bcea6f..6bd7a0241c 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -75,4 +75,11 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); } + + @Test + public void testMultisetCondition() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("WhereClauseMultiset.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseMultiset.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseMultiset.pls new file mode 100644 index 0000000000..29da118eb9 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseMultiset.pls @@ -0,0 +1,35 @@ +-- +-- Where Clause With Multiset Conditions +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Multiset-Conditions.html#GUID-E8164A15-715A-40A0-944D-26DF4C84DE3F +-- + +BEGIN + +SELECT customer_id, cust_address_ntab + INTO test + FROM customers_demo + WHERE cust_address_ntab IS A SET + ORDER BY customer_id; + +SELECT product_id, TO_CHAR(ad_finaltext) AS text + INTO test + FROM print_media + WHERE ad_textdocs_ntab IS NOT EMPTY + ORDER BY product_id, text; + +SELECT customer_id, cust_address_ntab + INTO test + FROM customers_demo + WHERE cust_address_typ('8768 N State Rd 37', 47404, + 'Bloomington', 'IN', 'US') + MEMBER OF cust_address_ntab + ORDER BY customer_id; + +SELECT customer_id, cust_address_ntab + INTO test + FROM customers_demo + WHERE cust_address_ntab SUBMULTISET OF cust_address2_ntab + ORDER BY customer_id; + +END; +/ From b32e5b09108a54e4bc2e6dd8c5bcf3c1c98ecb84 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 24 Jan 2019 20:14:06 +0100 Subject: [PATCH 59/90] [plsql] Support REGEXP_LIKE condition --- pmd-plsql/etc/grammar/PldocAST.jjt | 18 ++++++++- pmd-plsql/src/main/ant/alljavacc.xml | 1 + .../plsql/ast/ASTRegexpLikeCondition.java | 40 +++++++++++++++++++ .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 +++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 +++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 12 ++++++ .../lang/plsql/ast/WhereClauseRegexpLike.pls | 20 ++++++++++ 7 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/ASTRegexpLikeCondition.java create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseRegexpLike.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index c61770b02d..9d7327454c 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1310,6 +1310,8 @@ void Condition2() #void : LOOKAHEAD(9) InCondition() | LOOKAHEAD(9) LikeCondition() + | + LOOKAHEAD(9) RegexpLikeCondition() | LOOKAHEAD(4) BetweenCondition() | @@ -1342,6 +1344,19 @@ ASTLikeCondition LikeCondition() : { return jjtThis; } } +ASTRegexpLikeCondition RegexpLikeCondition() : +{ + Token matchParam = null; +} +{ + "(" + ID() "," + StringLiteral() + [ "," ( matchParam = | matchParam = ) {jjtThis.setMatchParam(matchParam.toString());} ] + ")" + { return jjtThis; } +} + ASTExistsCondition ExistsCondition() : {} { @@ -4365,6 +4380,7 @@ TOKEN [IGNORE_CASE]: | | | + | | | | //SRT 2011-04-17 @@ -5523,7 +5539,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -//| +| //| //| //| diff --git a/pmd-plsql/src/main/ant/alljavacc.xml b/pmd-plsql/src/main/ant/alljavacc.xml index dbd24f2f2c..3f934511da 100644 --- a/pmd-plsql/src/main/ant/alljavacc.xml +++ b/pmd-plsql/src/main/ant/alljavacc.xml @@ -62,6 +62,7 @@ + diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/ASTRegexpLikeCondition.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/ASTRegexpLikeCondition.java new file mode 100644 index 0000000000..cb4f588c82 --- /dev/null +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/ASTRegexpLikeCondition.java @@ -0,0 +1,40 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.plsql.ast; + +/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=true,VISITOR=true,TRACK_TOKENS=false,NODE_PREFIX=AST,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ +public class ASTRegexpLikeCondition extends net.sourceforge.pmd.lang.plsql.ast.AbstractPLSQLNode { + private String matchParam; + + public ASTRegexpLikeCondition(int id) { + super(id); + } + + public ASTRegexpLikeCondition(PLSQLParser p, int id) { + super(p, id); + } + + @Override + public Object jjtAccept(PLSQLParserVisitor visitor, Object data) { + return visitor.visit(this, data); + } + + void setMatchParam(String matchParam) { + this.matchParam = matchParam; + } + + public String getMatchParam() { + return this.matchParam; + } + + public ASTID getSourceChar() { + return (ASTID) jjtGetChild(0); + } + + public ASTStringLiteral getPattern() { + return (ASTStringLiteral) jjtGetChild(1); + } +} +/* JavaCC - OriginalChecksum=afb8806a0c67f95b736d6e8bc46def15 (do not edit this line) */ diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 770b089c2a..4c6e564504 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -1011,4 +1011,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTSubmultisetCondition node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTRegexpLikeCondition node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index 66f58346c7..89da4e2582 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1105,6 +1105,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTRegexpLikeCondition node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index 6bd7a0241c..98ef0c72ff 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -82,4 +82,16 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); } + + @Test + public void testRegexpLikeCondition() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("WhereClauseRegexpLike.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + List regexps = input.findDescendantsOfType(ASTRegexpLikeCondition.class); + Assert.assertEquals(2, regexps.size()); + Assert.assertEquals("last_name", regexps.get(1).getSourceChar().getImage()); + Assert.assertEquals("'([aeiou])\\1'", regexps.get(1).getPattern().getImage()); + Assert.assertEquals("'i'", regexps.get(1).getMatchParam()); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseRegexpLike.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseRegexpLike.pls new file mode 100644 index 0000000000..7345ee5ad0 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseRegexpLike.pls @@ -0,0 +1,20 @@ +-- +-- Where Clause With Regexp Like +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Pattern-matching-Conditions.html#GUID-D2124F3A-C6E4-4CCA-A40E-2FFCABFD8E19 +-- + +BEGIN + +SELECT first_name, last_name +INTO test +FROM employees +WHERE REGEXP_LIKE (first_name, '^Ste(v|ph)en$') +ORDER BY first_name, last_name; + +SELECT last_name +INTO test +FROM employees +WHERE REGEXP_LIKE (last_name, '([aeiou])\1', 'i') +ORDER BY last_name; +END; +/ From 833119e00c583228c81856d3074b210d54e9ee19 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 10 Feb 2019 10:35:21 +0100 Subject: [PATCH 60/90] [plsql] Fix test cases * table is a reserved word and cannot be used as an identifier * correctly name test class for TableCollectionExpressionTest --- ...tionExpression.java => TableCollectionExpressionTest.java} | 2 +- .../sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/{TableCollectionExpression.java => TableCollectionExpressionTest.java} (92%) diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpression.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpressionTest.java similarity index 92% rename from pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpression.java rename to pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpressionTest.java index e5ddb26302..1daad5882a 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpression.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/TableCollectionExpressionTest.java @@ -12,7 +12,7 @@ import org.junit.Test; import net.sourceforge.pmd.lang.plsql.AbstractPLSQLParserTst; -public class TableCollectionExpression extends AbstractPLSQLParserTst { +public class TableCollectionExpressionTest extends AbstractPLSQLParserTst { @Test public void testExamples() throws Exception { diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls index 6c67d24e65..a6c25d36d5 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseSubqueries.pls @@ -4,10 +4,10 @@ BEGIN -SELECT id INTO v_id FROM table +SELECT id INTO v_id FROM my_table WHERE id = (SELECT id FROM other_table); -UPDATE table SET name = 'a' +UPDATE my_table SET name = 'a' WHERE id = (SELECT id FROM other_table); END; From f049a36e5cccb1881971dc7b8d74282afd900466 Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Sat, 16 Feb 2019 00:24:16 -0300 Subject: [PATCH 61/90] Refactored token filter --- .../sourceforge/pmd/cpd/AntlrTokenizer.java | 7 +- .../pmd/cpd/token/AntlrHiddenTokenFilter.java | 16 ---- .../pmd/cpd/token/AntlrTokenFilter.java | 39 +++++++- .../pmd/cpd/token/BaseTokenFilter.java | 92 +++++++++++++++++++ .../pmd/cpd/token/JavaCCTokenFilter.java | 64 +------------ 5 files changed, 132 insertions(+), 86 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index eb9001e0d3..22a2b5d5c5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -8,13 +8,10 @@ import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; import net.sourceforge.pmd.cpd.token.AntlrToken; -import net.sourceforge.pmd.cpd.token.AntlrHiddenTokenFilter; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import static org.antlr.v4.runtime.Token.EOF; - /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. */ @@ -30,7 +27,7 @@ public abstract class AntlrTokenizer implements Tokenizer { try { AntlrToken currentToken = tokenFilter.getNextToken(); - while (currentToken != null && currentToken.getType() != EOF) { + while (currentToken != null) { processToken(tokenEntries, tokenManager.getFileName(), currentToken); currentToken = tokenFilter.getNextToken(); } @@ -46,7 +43,7 @@ public abstract class AntlrTokenizer implements Tokenizer { } protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { - return new AntlrHiddenTokenFilter(tokenManager); + return new AntlrTokenFilter(tokenManager); } /* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java deleted file mode 100644 index 27acac6d34..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrHiddenTokenFilter.java +++ /dev/null @@ -1,16 +0,0 @@ -package net.sourceforge.pmd.cpd.token; - -import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; - -public class AntlrHiddenTokenFilter extends AntlrTokenFilter { - - public AntlrHiddenTokenFilter(final AntlrTokenManager tokenManager) { - super(tokenManager); - } - - @Override - public AntlrToken getNextToken() { - AntlrToken token = super.getNextToken(); - return token.isHidden() ? getNextToken() : token; - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java index 74ae610bef..e15aff8b34 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -1,17 +1,46 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + package net.sourceforge.pmd.cpd.token; +import static org.antlr.v4.runtime.Token.EOF; + import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; -public class AntlrTokenFilter implements TokenFilter { +/** + * A generic filter for Antlr-based token managers that allows to use comments + * to enable / disable analysis of parts of the stream + */ +public class AntlrTokenFilter extends BaseTokenFilter { - private TokenFilter tokenFilter; + private boolean discardingHiddenTokens = false; + /** + * Creates a new AntlrTokenFilter + * @param tokenManager The token manager from which to retrieve tokens to be filtered + */ public AntlrTokenFilter(final AntlrTokenManager tokenManager) { - this.tokenFilter = new JavaCCTokenFilter(tokenManager); + super(tokenManager); } @Override - public AntlrToken getNextToken() { - return (AntlrToken) tokenFilter.getNextToken(); + protected boolean shouldStopProcessing(final AntlrToken currentToken) { + return currentToken.getType() == EOF; + } + + @Override + protected void analyzeToken(final AntlrToken currentToken) { + super.analyzeToken(currentToken); + analyzeHiddenTokens(currentToken); + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return super.isLanguageSpecificDiscarding() || discardingHiddenTokens; + } + + private void analyzeHiddenTokens(final AntlrToken token) { + discardingHiddenTokens = token.isHidden(); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java new file mode 100644 index 0000000000..5fe73fdd2a --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java @@ -0,0 +1,92 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd.token; + +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.GenericToken; + +/** + * A generic filter for PMD token managers that allows to use comments + * to enable / disable analysis of parts of the stream + */ +public abstract class BaseTokenFilter implements TokenFilter { + + private final TokenManager tokenManager; + private boolean discardingSuppressing; + + /** + * Creates a new BaseTokenFilter + * @param tokenManager The token manager from which to retrieve tokens to be filtered + */ + public BaseTokenFilter(final TokenManager tokenManager) { + this.tokenManager = tokenManager; + } + + @Override + public final T getNextToken() { + T currentToken = (T) tokenManager.getNextToken(); + while (!shouldStopProcessing(currentToken)) { + analyzeToken(currentToken); + processCPDSuppression(currentToken); + + if (!isDiscarding()) { + return currentToken; + } + + currentToken = (T) tokenManager.getNextToken(); + } + + return null; + } + + private boolean isDiscarding() { + return discardingSuppressing || isLanguageSpecificDiscarding(); + } + + private void processCPDSuppression(final T currentToken) { + // Check if a comment is altering the suppression state + GenericToken comment = currentToken.getPreviousComment(); + while (comment != null) { + if (comment.getImage().contains("CPD-OFF")) { + discardingSuppressing = true; + break; + } + if (comment.getImage().contains("CPD-ON")) { + discardingSuppressing = false; + break; + } + comment = comment.getPreviousComment(); + } + } + + /** + * Extension point for subclasses to indicate when to stop filtering tokens. + * + * @param currentToken The token to be analyzed + * @return True if the token filter has finished consuming all tokens, false otherwise + */ + protected abstract boolean shouldStopProcessing(T currentToken); + + /** + * Extension point for subclasses to indicate tokens are to be filtered. + * + * @return True if tokens should be filtered, false otherwise + */ + protected boolean isLanguageSpecificDiscarding() { + return false; + } + + /** + * Extension point for subclasses to analyze all tokens (before filtering) + * and update internal status to decide on custom discard rules. + * + * @param currentToken The token to be analyzed + * @see #isLanguageSpecificDiscarding() + */ + protected void analyzeToken(final T currentToken) { + // noop + } + +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java index 0b1cd53b4e..734b9fcff3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java @@ -11,74 +11,18 @@ import net.sourceforge.pmd.lang.ast.GenericToken; * A generic filter for JavaCC-based token managers that allows to use comments * to enable / disable analysis of parts of the stream */ -public class JavaCCTokenFilter implements TokenFilter { +public class JavaCCTokenFilter extends BaseTokenFilter { - private final TokenManager tokenManager; - private boolean discardingSuppressing; - /** * Creates a new JavaCCTokenFilter * @param tokenManager The token manager from which to retrieve tokens to be filtered */ public JavaCCTokenFilter(final TokenManager tokenManager) { - this.tokenManager = tokenManager; + super(tokenManager); } @Override - public final GenericToken getNextToken() { - GenericToken currentToken = (GenericToken) tokenManager.getNextToken(); - while (!currentToken.getImage().isEmpty()) { - analyzeToken(currentToken); - processCPDSuppression(currentToken); - - if (!isDiscarding()) { - return currentToken; - } - - currentToken = (GenericToken) tokenManager.getNextToken(); - } - - return null; + protected boolean shouldStopProcessing(final GenericToken currentToken) { + return currentToken.getImage().isEmpty(); } - - private boolean isDiscarding() { - return discardingSuppressing || isLanguageSpecificDiscarding(); - } - - private void processCPDSuppression(final GenericToken currentToken) { - // Check if a comment is altering the suppression state - GenericToken comment = currentToken.getPreviousComment(); - while (comment != null) { - if (comment.getImage().contains("CPD-OFF")) { - discardingSuppressing = true; - break; - } - if (comment.getImage().contains("CPD-ON")) { - discardingSuppressing = false; - break; - } - comment = comment.getPreviousComment(); - } - } - - /** - * Extension point for subclasses to indicate tokens are to be filtered. - * - * @return True if tokens should be filtered, false otherwise - */ - protected boolean isLanguageSpecificDiscarding() { - return false; - } - - /** - * Extension point for subclasses to analyze all tokens (before filtering) - * and update internal status to decide on custom discard rules. - * - * @param currentToken The token to be analyzed - * @see #isLanguageSpecificDiscarding() - */ - protected void analyzeToken(final GenericToken currentToken) { - // noop - } - } From 3cff4f3d22fbd6c1dc6aa15140b2c4a9c63609d5 Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Sat, 16 Feb 2019 01:19:05 -0300 Subject: [PATCH 62/90] Minor fixes --- .../sourceforge/pmd/cpd/AntlrTokenizer.java | 8 ++++---- .../pmd/cpd/token/AntlrTokenFilter.java | 1 - .../pmd/cpd/token/BaseTokenFilter.java | 20 +++++++++---------- .../pmd/cpd/token/JavaCCTokenFilter.java | 5 +++++ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index 22a2b5d5c5..2b0d2c56ea 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -22,8 +22,8 @@ public abstract class AntlrTokenizer implements Tokenizer { @Override public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) { - AntlrTokenManager tokenManager = getLexerForSource(sourceCode); - AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager); + final AntlrTokenManager tokenManager = getLexerForSource(sourceCode); + final AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager); try { AntlrToken currentToken = tokenFilter.getNextToken(); @@ -42,7 +42,7 @@ public abstract class AntlrTokenizer implements Tokenizer { } } - protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { return new AntlrTokenFilter(tokenManager); } @@ -51,7 +51,7 @@ public abstract class AntlrTokenizer implements Tokenizer { return CharStreams.fromString(buffer.toString()); } - private void processToken(Tokens tokenEntries, String fileName, AntlrToken token) { + private void processToken(final Tokens tokenEntries, final String fileName, final AntlrToken token) { final TokenEntry tokenEntry = new TokenEntry(token.getImage(), fileName, token.getBeginLine()); tokenEntries.add(tokenEntry); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java index e15aff8b34..e1eee7b77b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -31,7 +31,6 @@ public class AntlrTokenFilter extends BaseTokenFilter { @Override protected void analyzeToken(final AntlrToken currentToken) { - super.analyzeToken(currentToken); analyzeHiddenTokens(currentToken); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java index 5fe73fdd2a..4be453b919 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java @@ -61,14 +61,6 @@ public abstract class BaseTokenFilter implements TokenFi } } - /** - * Extension point for subclasses to indicate when to stop filtering tokens. - * - * @param currentToken The token to be analyzed - * @return True if the token filter has finished consuming all tokens, false otherwise - */ - protected abstract boolean shouldStopProcessing(T currentToken); - /** * Extension point for subclasses to indicate tokens are to be filtered. * @@ -78,6 +70,14 @@ public abstract class BaseTokenFilter implements TokenFi return false; } + /** + * Extension point for subclasses to indicate when to stop filtering tokens. + * + * @param currentToken The token to be analyzed + * @return True if the token filter has finished consuming all tokens, false otherwise + */ + protected abstract boolean shouldStopProcessing(T currentToken); + /** * Extension point for subclasses to analyze all tokens (before filtering) * and update internal status to decide on custom discard rules. @@ -85,8 +85,6 @@ public abstract class BaseTokenFilter implements TokenFi * @param currentToken The token to be analyzed * @see #isLanguageSpecificDiscarding() */ - protected void analyzeToken(final T currentToken) { - // noop - } + protected abstract void analyzeToken(T currentToken); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java index 734b9fcff3..3231e353e2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java @@ -25,4 +25,9 @@ public class JavaCCTokenFilter extends BaseTokenFilter { protected boolean shouldStopProcessing(final GenericToken currentToken) { return currentToken.getImage().isEmpty(); } + + @Override + protected void analyzeToken(final GenericToken currentToken) { + // noop + } } From 72f2e425b278ea7cf008b664aa54134071e8338d Mon Sep 17 00:00:00 2001 From: lsoncini Date: Sat, 16 Feb 2019 15:49:03 -0300 Subject: [PATCH 63/90] KotlinTokenizer refactor --- .../main/java/net/sourceforge/pmd/PMD.java | 6 +- .../sourceforge/pmd/cpd/KotlinTokenizer.java | 75 +++++++++++-------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java index 7372f15906..c78e191d08 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java @@ -316,7 +316,7 @@ public class PMD { // Make sure the cache is listening for analysis results ctx.getReport().addListener(configuration.getAnalysisCache()); - final RuleSetFactory silentFactoy = new RuleSetFactory(ruleSetFactory, false); + final RuleSetFactory silentFactory = new RuleSetFactory(ruleSetFactory, false); /* * Check if multithreaded support is available. ExecutorService can also @@ -324,9 +324,9 @@ public class PMD { * "-threads 0" command line option. */ if (configuration.getThreads() > 0) { - new MultiThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MultiThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } else { - new MonoThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers); + new MonoThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers); } // Persist the analysis cache diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index 17da5ec802..b4ee895e5f 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -5,10 +5,9 @@ package net.sourceforge.pmd.cpd; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; @@ -17,46 +16,56 @@ import net.sourceforge.pmd.lang.kotlin.antlr4.Kotlin; */ public class KotlinTokenizer extends AntlrTokenizer { - private boolean discardingPackageAndImport = false; - @Override protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); - final Lexer lexer = new Kotlin(charStream); - final AntlrTokenManager tokenManager = new AntlrTokenManager(lexer, sourceCode.getFileName()) { - @Override - public Object getNextToken() { - AntlrToken nextToken; - boolean done = false; - do { - nextToken = (AntlrToken) super.getNextToken(); - analyzeTokenStart(nextToken); - if (!nextToken.isHidden() && nextToken.getType() != Kotlin.NL && !isDiscarding()) { - done = true; - } - analyzeTokenEnd(nextToken); - } while (!done && nextToken.getType() != Token.EOF); - return nextToken; - } - }; - return tokenManager; + return new AntlrTokenManager(new Kotlin(charStream), sourceCode.getFileName()); } - private boolean isDiscarding() { - return discardingPackageAndImport; + @Override + protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + return new KotlinTokenFilter(tokenManager); } - private void analyzeTokenStart(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { - discardingPackageAndImport = true; + /** + * The {@link KotlinTokenFilter} extends the {@link AntlrTokenFilter} to discard + * Kotlin-specific tokens. + *

+ * By default, it discards package and import statements, and + * enables annotation-based CPD suppression. + *

+ */ + private static class KotlinTokenFilter extends AntlrTokenFilter { + private boolean discardingPackageAndImport = false; + private boolean discardingNL = false; + + /* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) { + super(tokenManager); } - } - private void analyzeTokenEnd(final AntlrToken currentToken) { - final int type = currentToken.getType(); - if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { - discardingPackageAndImport = false; + @Override + protected void analyzeToken(final AntlrToken currentToken) { + super.analyzeToken(currentToken); + skipPackageAndImport(currentToken); + skipNewLines(currentToken); + } + + private void skipPackageAndImport(final AntlrToken currentToken) { + final int type = currentToken.getType(); + if (type == Kotlin.PACKAGE || type == Kotlin.IMPORT) { + discardingPackageAndImport = true; + } else if (discardingPackageAndImport && (type == Kotlin.SEMICOLON || type == Kotlin.NL)) { + discardingPackageAndImport = false; + } + } + + private void skipNewLines(final AntlrToken currentToken) { + discardingNL = currentToken.getType() == Kotlin.NL; + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return discardingPackageAndImport || discardingNL || super.isLanguageSpecificDiscarding(); } } } From d56bdc1c84f1589459b7462e7ee636be131f6455 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 13 Feb 2019 19:34:18 +0100 Subject: [PATCH 64/90] [plsql] Support AnalyticClause --- pmd-plsql/etc/grammar/PldocAST.jjt | 36 ++++++++++++++++--- .../plsql/ast/PLSQLParserVisitorAdapter.java | 10 ++++++ .../lang/plsql/rule/AbstractPLSQLRule.java | 10 ++++++ .../pmd/lang/plsql/ast/SelectExpressions.pls | 7 ++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index ac8ea5a336..8ef66b2b11 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1532,6 +1532,35 @@ ASTFunctionName FunctionName() : } } +/** + * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Analytic-Functions.html + */ +ASTAnalyticClause AnalyticClause() : +{} +{ + "OVER" "(" [ QueryPartitionClause() ] [ OrderByClause() [ WindowingClause() ] ] ")" + { return jjtThis; } +} + +/** + * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Analytic-Functions.html + */ +ASTWindowingClause WindowingClause() : +{} +{ + ( | ) + ( + ( + ( LOOKAHEAD(3) "UNBOUNDED" "PRECEDING" | LOOKAHEAD(2) | SqlExpression() ( "PRECEDING" | "FOLLOWING" ) ) + + ( LOOKAHEAD(3) "UNBOUNDED" "FOLLOWING" | LOOKAHEAD(2) | SqlExpression() ( "PRECEDING" | "FOLLOWING" ) ) + ) + | + ( LOOKAHEAD(3) "UNBOUNDED" "PRECEDING" | LOOKAHEAD(2) | SqlExpression() "PRECEDING" ) + ) + { return jjtThis; } +} + ASTColumn Column() : { ASTID id; } { @@ -2921,7 +2950,8 @@ ASTPrimaryPrefix PrimaryPrefix() : } { ( - LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() + // Note: AnalyticClause is only allowed for specific functions, but this grammar allows it for all functions. + LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() [ AnalyticClause() ] | LOOKAHEAD(Literal()) simpleNode = Literal() { sb.append(simpleNode.getImage()) ; } | LOOKAHEAD(MultiSetCondition()) simpleNode = MultiSetCondition() | LOOKAHEAD(TrimExpression()) simpleNode = TrimExpression() //SRT 20110613.3 @@ -4780,7 +4810,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} { // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' ( -"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | +"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | | | @@ -5898,7 +5928,6 @@ ASTID ID(): {} //20120429 | | //| // | - | //RESERVED WORD | | //SYNTAX | @@ -6144,7 +6173,6 @@ ASTQualifiedID QualifiedID(): {} //20120429 | //| // - // | | | diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 76e40c8a77..d016edaa1f 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -1026,4 +1026,14 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTFunctionName node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTAnalyticClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTWindowingClause node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index b52cdb4311..6057471142 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1120,6 +1120,16 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTAnalyticClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTWindowingClause node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index a8084fd395..5d7185bf9a 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -17,6 +17,13 @@ SELECT INTO some_record FROM some_table; +-- Example from: https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/AVG.html +SELECT manager_id, last_name, hire_date, salary, + AVG(salary) OVER (PARTITION BY manager_id ORDER BY hire_date + ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS c_mavg + INTO some_record + FROM employees + ORDER BY manager_id, hire_date, salary; END; / \ No newline at end of file From 892b53dc6ff7caa4d480cd40c2bf2eb7a83984ce Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 13 Feb 2019 19:53:31 +0100 Subject: [PATCH 65/90] [plsql] Support WithinClause for LISTAGG function (and similar) --- pmd-plsql/etc/grammar/PldocAST.jjt | 21 ++++++++++++++----- .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 +++++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 +++++ .../pmd/lang/plsql/ast/SelectExpressions.pls | 15 +++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 8ef66b2b11..a50c8633b8 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1561,6 +1561,19 @@ ASTWindowingClause WindowingClause() : { return jjtThis; } } +/** + * Within Clause is used for the following analytic functions: CUME_DIST, DENSE_RANK, LISTAGG, PERCENT_RANK, PERCENTILE_CONT, + * PERCENTILE_DISC, RANK. + * + * See e.g. https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/LISTAGG.html + */ +ASTWithinClause WithinClause() : +{} +{ + "WITHIN" "(" OrderByClause() ")" [ LOOKAHEAD(2) "OVER" "(" QueryPartitionClause() ")" ] + { return jjtThis; } +} + ASTColumn Column() : { ASTID id; } { @@ -2950,8 +2963,8 @@ ASTPrimaryPrefix PrimaryPrefix() : } { ( - // Note: AnalyticClause is only allowed for specific functions, but this grammar allows it for all functions. - LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() [ AnalyticClause() ] + // Note: AnalyticClause and WithinClause are only allowed for specific functions, but this grammar allows it for all functions. + LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() [ AnalyticClause() ] [ WithinClause() ] | LOOKAHEAD(Literal()) simpleNode = Literal() { sb.append(simpleNode.getImage()) ; } | LOOKAHEAD(MultiSetCondition()) simpleNode = MultiSetCondition() | LOOKAHEAD(TrimExpression()) simpleNode = TrimExpression() //SRT 20110613.3 @@ -4810,7 +4823,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} { // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' ( -"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | +"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | "WITHIN" | | | @@ -5974,7 +5987,6 @@ ASTID ID(): {} // | | // this causes bug 643043 Procedure w/o params appears as variable | //SYNTAX - //| //RESERVED WORD | //RESERVED WORD //20120501 | | //SYNTAX @@ -6221,7 +6233,6 @@ ASTQualifiedID QualifiedID(): {} // //| // - // // //20120501 | // diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index d016edaa1f..4d1815fe7f 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -1036,4 +1036,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTWindowingClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTWithinClause node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index 6057471142..dd57566cbf 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1130,6 +1130,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTWithinClause node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index 5d7185bf9a..add6138f82 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -25,5 +25,20 @@ SELECT manager_id, last_name, hire_date, salary, FROM employees ORDER BY manager_id, hire_date, salary; +-- Example from: https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/LISTAGG.html +SELECT LISTAGG(last_name, '; ') + WITHIN GROUP (ORDER BY hire_date, last_name) "Emp_list", + MIN(hire_date) "Earliest" + INTO some_record + FROM employees + WHERE department_id = 30; + +SELECT department_id "Dept.", + LISTAGG(last_name, '; ') WITHIN GROUP (ORDER BY hire_date) "Employees" + INTO some_record + FROM employees + GROUP BY department_id + ORDER BY department_id; + END; / \ No newline at end of file From 4814ece9acd3ffdf21f0215179d66247676029f5 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 16 Feb 2019 20:12:38 +0100 Subject: [PATCH 66/90] [plsql] Support ListaggOverflowClause for LISTAGG function --- pmd-plsql/etc/grammar/PldocAST.jjt | 35 +++++++++++-------- .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 +++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 +++ .../pmd/lang/plsql/ast/SelectExpressions.pls | 16 +++++++++ 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index a50c8633b8..8ff36db203 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1574,6 +1574,18 @@ ASTWithinClause WithinClause() : { return jjtThis; } } +ASTListaggOverflowClause ListaggOverflowClause() : +{} +{ + "OVERFLOW" + ( + "ERROR" + | + [ StringLiteral() ] [ ( | "WITHOUT" ) "COUNT" ] + ) + { return jjtThis; } +} + ASTColumn Column() : { ASTID id; } { @@ -3222,15 +3234,15 @@ ASTQualifiedName QualifiedName() : } ASTArguments Arguments() : +{} { -} -{ - //"(" [ ArgumentList() ] ")" + "(" + [ [ | | ] ("*" | ArgumentList()) ] + // Note: ListaggOverflowClause is only allowed for LISTAGG function, but this grammar allows it for all functions. + [ ListaggOverflowClause() ] + ")" - "(" [ [ | | ] ("*" | ArgumentList()) ] ")" - { - return jjtThis ; - } + { return jjtThis; } } ASTArgumentList ArgumentList() : @@ -4824,6 +4836,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' ( "FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | "WITHIN" | +"OVERFLOW" | "ERROR" | "WITHOUT" | "COUNT" | | | @@ -5072,7 +5085,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| //| //| | @@ -5491,7 +5503,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| //| //| | @@ -5877,7 +5888,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} | //| //| -//| //| //| //| @@ -6021,8 +6031,6 @@ ASTID ID(): {} | //RESERVED WORD | | - //| //RESERVED WORD - //| //RESERVED WORD //20120501 | | | | @@ -6088,7 +6096,6 @@ ASTID ID(): {} | //| | | //20120429 | | | | - | //RESERVED WORD | //SYNTAX // | | | //| | @@ -6279,7 +6286,6 @@ ASTQualifiedID QualifiedID(): {} | | // - // //20120501 | //20120501 | | @@ -6363,7 +6369,6 @@ ASTQualifiedID QualifiedID(): {} //20120429 | //20120429 | //| - // | // | //| diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 4d1815fe7f..8ed2eef88e 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -1041,4 +1041,9 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTWithinClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTListaggOverflowClause node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index dd57566cbf..dfae6e8e25 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1135,6 +1135,11 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTListaggOverflowClause node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index add6138f82..581e689b3a 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -40,5 +40,21 @@ SELECT department_id "Dept.", GROUP BY department_id ORDER BY department_id; +SELECT department_id "Dept.", + LISTAGG(last_name, '; ' ON OVERFLOW TRUNCATE '...') + WITHIN GROUP (ORDER BY hire_date) "Employees" + INTO some_record + FROM employees + GROUP BY department_id + ORDER BY department_id; + +SELECT department_id "Dept", hire_date "Date", last_name "Name", + LISTAGG(last_name, '; ') WITHIN GROUP (ORDER BY hire_date, last_name) + OVER (PARTITION BY department_id) as "Emp_list" + INTO some_record + FROM employees + WHERE hire_date < '01-SEP-2003' + ORDER BY "Dept", "Date", "Name"; + END; / \ No newline at end of file From f43e261243cd51369441b29202150a42f2287d3d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 13 Feb 2019 20:32:46 +0100 Subject: [PATCH 67/90] [plsql] Support more than simple names for order by and select into. --- pmd-plsql/etc/grammar/PldocAST.jjt | 14 +++++++++----- .../lang/plsql/ast/SelectIntoStatementTest.java | 7 +++++++ .../pmd/lang/plsql/ast/SelectExpressions.pls | 4 ++++ .../plsql/ast/SelectIntoStatementRecordField.pls | 13 +++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementRecordField.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 8ff36db203..f2c918fd30 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1189,9 +1189,9 @@ ASTOrderByClause OrderByClause() : void OrderByEntry() #void : {} { - ( LOOKAHEAD(2) ColumnAlias() | LOOKAHEAD(2) SqlExpression() ) + ( LOOKAHEAD(ID() ".") SqlExpression() | LOOKAHEAD(2) ColumnAlias() | SqlExpression() ) [ | ] - [ LOOKAHEAD(2) "FIRST" | LOOKAHEAD(2) "LAST" ] + [ ( "FIRST" | "LAST" ) ] } /** @@ -1675,10 +1675,14 @@ ASTIntoClause IntoClause() : } ASTVariableName VariableName() : -{ ASTID id; } +{ ASTID id; StringBuilder name = new StringBuilder(); } { - id = ID() {jjtThis.setImage(id.getImage());} - { return jjtThis; } + id = ID() { name.append(id.getImage()); } + [ "." id = ID() { name.append('.').append(id.getImage()); } ] + { + jjtThis.setImage(name.toString()); + return jjtThis; + } } ASTBulkCollectIntoClause BulkCollectIntoClause() : diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementTest.java index 788fcda7ab..61b192fc2d 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementTest.java @@ -61,4 +61,11 @@ public class SelectIntoStatementTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); } + + @Test + public void testParsingIntoRecordField() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("SelectIntoStatementRecordField.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index 581e689b3a..1f2921da36 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -56,5 +56,9 @@ SELECT department_id "Dept", hire_date "Date", last_name "Name", WHERE hire_date < '01-SEP-2003' ORDER BY "Dept", "Date", "Name"; +SELECT listagg(e.email,',') within group (order by e.email )INTO + v_task_resp + FROM sso_auth_employees e; + END; / \ No newline at end of file diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementRecordField.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementRecordField.pls new file mode 100644 index 0000000000..61d8795f8d --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectIntoStatementRecordField.pls @@ -0,0 +1,13 @@ +-- +-- BSD-style license; for more info see http://pmd.sourceforge.net/license.html +-- + +BEGIN + +SELECT the_id + INTO my_record.the_id + FROM my_table + WHERE the_id = '1'; + +END; +/ \ No newline at end of file From e8ed43d7e23b3ea15393b6cf350a73bdee4d7761 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 14 Feb 2019 21:44:43 +0100 Subject: [PATCH 68/90] [plsql] Fix parsing of union in SELECT INTO statements Also allow "normal" SELECT statement (without INTO). --- pmd-plsql/etc/grammar/PldocAST.jjt | 12 +++++++- .../pmd/lang/plsql/ast/SelectUnionTest.java | 25 ++++++++++++++++ .../pmd/lang/plsql/ast/SelectUnion.pls | 30 +++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectUnionTest.java create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectUnion.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index f2c918fd30..6227d4da50 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1147,6 +1147,9 @@ void RestOfStatement() #void : [ WhereClause() ] [ HierarchicalQueryClause() ] [ GroupByClause() ] + + (LOOKAHEAD(2) ( SubqueryOperation() ) Subquery() )* + [ OrderByClause() ] [ RowLimitingClause() ] } @@ -1950,7 +1953,14 @@ ASTUnlabelledStatement UnlabelledStatement() : {} { ( - SelectIntoStatement() ";" | + // small optimization: SelectIntoStatement and SelectStatement both begin with SELECT + // but to distinguish the two, a complete lookahead of SelectIntoStatement needs to be parsed. + // Using a lookahead of a single token first avoids this the syntatic lookahead for all other choices + // not related to SELECT statements. + LOOKAHEAD(||||
||) SkipPastNextTokenOccurrence(SQLPLUS_TERMINATOR) //Ignore SQL statements in scripts + | SelectStatement() (";")? + |(|||
||) SkipPastNextTokenOccurrence(SQLPLUS_TERMINATOR) //Ignore SQL statements in scripts ) ("/")* )* @@ -1303,7 +1304,7 @@ void Condition2() #void : { ( // a IS OF Type condition that starts with a function... - LOOKAHEAD(FunctionCall() ) IsOfTypeCondition() + LOOKAHEAD(FunctionCall() [] ) IsOfTypeCondition() | LOOKAHEAD(9) MultisetCondition() | @@ -2998,8 +2999,8 @@ ASTPrimaryPrefix PrimaryPrefix() : | LOOKAHEAD(ObjectExpression() ) ( simpleNode = ObjectExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110604 //| LOOKAHEAD(LikeExpression()) ( simpleNode = LikeExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110604 | ( simpleNode =Name() ) { sb.append(simpleNode.getImage()) ; } -| LOOKAHEAD() {sb.append("SELECT ..."); } Skip2NextTerminator(null,";") -| LOOKAHEAD(("(")+ ) "(" SelectStatement() ")" | LOOKAHEAD() () {sb.append("WITH ..."); } Skip2NextTerminator(null,";") | LOOKAHEAD(("(")+ ) ("(") {sb.append("(WITH ..."); } Skip2NextTerminator("(",")") ")" | diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/JoinClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/JoinClauseTest.java index 5d7c306f98..a86ef0d759 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/JoinClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/JoinClauseTest.java @@ -32,7 +32,7 @@ public class JoinClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); List joins = input.findDescendantsOfType(ASTInnerCrossJoinClause.class); - Assert.assertEquals(1, joins.size()); + Assert.assertEquals(2, joins.size()); Assert.assertFalse(joins.get(0).isCross()); Assert.assertTrue(joins.get(0).isNatural()); } diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java index a988f0b9bd..58eee511ea 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/SelectExpressionsTest.java @@ -29,4 +29,12 @@ public class SelectExpressionsTest extends AbstractPLSQLParserTst { ASTInput input = parsePLSQL(code); Assert.assertNotNull(input); } + + @Test + public void parseSelectSubqueryExpression() throws Exception { + String code = IOUtils.toString(this.getClass().getResourceAsStream("SelectSubqueryExpressions.pls"), + StandardCharsets.UTF_8); + ASTInput input = parsePLSQL(code); + Assert.assertNotNull(input); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/InnerNaturalJoin.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/InnerNaturalJoin.pls index f757dad46a..cc63eab847 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/InnerNaturalJoin.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/InnerNaturalJoin.pls @@ -1,12 +1,13 @@ BEGIN + SELECT department_id AS d_e_dept_id, e.last_name INTO r_record FROM departments NATURAL JOIN employees; -END; SELECT department_id AS d_e_dept_id, e.last_name INTO r_record FROM departments NATURAL INNER JOIN employees; + END; / diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/IsOfType.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/IsOfType.pls index 6806a04214..cc9e78e872 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/IsOfType.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/IsOfType.pls @@ -17,7 +17,7 @@ IF ioFPOobj IS NOT OF TYPE (ONLY FPOGE_OBJ) THEN loFPOGE_OBJ:=treat(ioFPOobj AS FPOGE_OBJ); end if; -loFPOGE_OBJ:=SELECT A FROM persons p WHERE IS OF TYPE (employee_t); -loFPOGE_OBJ:=SELECT A FROM persons p WHERE IS NOT OF TYPE (ONLY employee_t, other_t); +loFPOGE_OBJ:=SELECT A FROM persons p WHERE VALUE(p) IS OF TYPE (employee_t); +loFPOGE_OBJ:=SELECT A FROM persons p WHERE VALUE(p) IS NOT OF TYPE (ONLY employee_t, other_t); end; \ No newline at end of file diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectSubqueryExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectSubqueryExpressions.pls new file mode 100644 index 0000000000..44f3782af8 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectSubqueryExpressions.pls @@ -0,0 +1,18 @@ +-- +-- Subqueries / Expressions in the SelectList +-- + +BEGIN + +SELECT (SELECT a FROM DUAL) INTO foo FROM DUAL; + +SELECT (SELECT MAX(a) FROM DUAL) INTO foo FROM DUAL; + +SELECT (AVG(sal)*2) INTO foo FROM bar; + +SELECT a.object_name, + (SELECT MAX(NVL(b.position, -1)) FROM DUAL) + FROM DUAL; + +END; +/ \ No newline at end of file From d81e708db05e4e3d3c45888431c75aa2274d128a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 15 Feb 2019 22:15:12 +0100 Subject: [PATCH 70/90] [plsql] Improved UPDATE statement --- pmd-plsql/etc/grammar/PldocAST.jjt | 61 ++++++++++++++++--- .../plsql/ast/PLSQLParserVisitorAdapter.java | 15 +++++ .../lang/plsql/rule/AbstractPLSQLRule.java | 15 +++++ .../lang/plsql/ast/UpdateStatementTest.java | 12 ++++ .../plsql/ast/UpdateStatementExample2.pls | 55 +++++++++++++++++ .../pmd/lang/plsql/ast/UpdateStatementRef.pls | 12 ++++ 6 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementRef.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index e51cd3bc17..3e4f6222bd 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -2147,6 +2147,8 @@ ASTUpdateStatement UpdateStatement() : DMLTableExpressionClause() [ LOOKAHEAD(2) TableAlias() ] UpdateSetClause() [ WhereClause() ] + [ ReturningClause() ] + [ ErrorLoggingClause() ] { return jjtThis; } } @@ -2156,7 +2158,7 @@ ASTDMLTableExpressionClause DMLTableExpressionClause() : ( TableCollectionExpression() | - LOOKAHEAD(2) [ LOOKAHEAD(2) SchemaName() "." ] TableName() + LOOKAHEAD(2) [ LOOKAHEAD(2) SchemaName() "." ] TableName() [ "@" ID() | PartitionExtensionClause() ] | [ ] "(" Subquery() [ SubqueryRestrictionClause() ] ")" ) @@ -2164,19 +2166,64 @@ ASTDMLTableExpressionClause DMLTableExpressionClause() : { return jjtThis; } } +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/UPDATE.html#GUID-027A462D-379D-4E35-8611-410F3AC8FDA5__CHDBBDEI + */ +ASTPartitionExtensionClause PartitionExtensionClause() : +{} +{ + ( | "SUBPARTITION") ( "(" ID() ")" | "(" ( ID() (",")? )+ ")" ) + { return jjtThis; } +} + +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/UPDATE.html#GUID-027A462D-379D-4E35-8611-410F3AC8FDA5__I2126876 + */ ASTUpdateSetClause UpdateSetClause() : {} { ( - LOOKAHEAD(3) "=" ID() + LOOKAHEAD(1) "=" ID() | - [ LOOKAHEAD(2) TableName() "." ] Column() "=" ( Expression() | <_DEFAULT> ) - ( "," [ LOOKAHEAD(2) TableName() "." ] Column() "=" ( Expression() | <_DEFAULT> ) )* + "VALUE" "(" TableAlias() ")" "=" ( LOOKAHEAD(1) "(" Subquery() ")" | Expression() ) + | + ( + ( + ( "(" ( [ LOOKAHEAD(2) TableName() "." ] Column() (",")? )+ ")" "=" "(" Subquery() ")" ) + | + ( [ LOOKAHEAD(2) TableName() "." ] Column() "=" ( LOOKAHEAD(1) "(" Subquery() ")" | Expression() | <_DEFAULT> ) ) + ) + (",")? + )+ ) { return jjtThis; } } +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/UPDATE.html#GUID-027A462D-379D-4E35-8611-410F3AC8FDA5__I2126358 + */ +ASTReturningClause ReturningClause() : +{} +{ + ( | ) ( Expression() (",")? )+ ( ID() (",")? )+ + { return jjtThis; } +} + +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/UPDATE.html#GUID-027A462D-379D-4E35-8611-410F3AC8FDA5__BCEEAAGC + */ +ASTErrorLoggingClause ErrorLoggingClause() : +{} +{ + "LOG" "ERRORS" + [ [ LOOKAHEAD(2) SchemaName() "." ] TableName() ] + [ "(" SqlExpression() ")" ] + [ "REJECT" ( NumericLiteral() | "UNLIMITED" ) ] + { return jjtThis; } +} + + ASTDeleteStatement DeleteStatement() : {} { @@ -4851,7 +4898,7 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' ( "FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | "WITHIN" | -"OVERFLOW" | "ERROR" | "WITHOUT" | "COUNT" | +"OVERFLOW" | "ERROR" | "WITHOUT" | "COUNT" | "VALUE" | "SUBPARTITION" | "LOG" | "ERRORS" | "REJECT" | "UNLIMITED" | | | @@ -5101,7 +5148,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| | //| //| @@ -5287,7 +5333,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| //| //| //| @@ -5632,7 +5677,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| | //| -//| //| //| //| @@ -5851,7 +5895,6 @@ ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} //| //| //| -//| //| //| //| diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 8ed2eef88e..d6c3cc80da 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -1046,4 +1046,19 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { public Object visit(ASTListaggOverflowClause node, Object data) { return visit((PLSQLNode) node, data); } + + @Override + public Object visit(ASTPartitionExtensionClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTReturningClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTErrorLoggingClause node, Object data) { + return visit((PLSQLNode) node, data); + } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index dfae6e8e25..50741ce082 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -1140,6 +1140,21 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } + @Override + public Object visit(ASTPartitionExtensionClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTReturningClause node, Object data) { + return visit((PLSQLNode) node, data); + } + + @Override + public Object visit(ASTErrorLoggingClause node, Object data) { + return visit((PLSQLNode) node, data); + } + /* * Treat all Executable Code */ diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementTest.java index 697290e1ce..f5abafb4f6 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementTest.java @@ -21,4 +21,16 @@ public class UpdateStatementTest extends AbstractPLSQLParserTst { Assert.assertEquals(2, updateStatements.get(1).getFirstChildOfType(ASTUpdateSetClause.class) .findChildrenOfType(ASTColumn.class).size()); } + + @Test + public void parseUpdateStatementExample2() { + ASTInput input = parsePLSQL(loadTestResource("UpdateStatementExample2.pls")); + Assert.assertNotNull(input); + } + + @Test + public void parseUpdateStatementRef() { + ASTInput input = parsePLSQL(loadTestResource("UpdateStatementRef.pls")); + Assert.assertNotNull(input); + } } diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls new file mode 100644 index 0000000000..02d8c76ad1 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls @@ -0,0 +1,55 @@ +-- +-- Examples from SQL Reference +-- https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/UPDATE.html +-- + +BEGIN + +UPDATE employees + SET commission_pct = NULL + WHERE job_id = 'SH_CLERK'; + +UPDATE employees SET + job_id = 'SA_MAN', salary = salary + 1000, department_id = 120 + WHERE first_name||' '||last_name = 'Douglas Grant'; + +UPDATE employees@remote + SET salary = salary*1.1 + WHERE last_name = 'Baer'; + +UPDATE employees a + SET department_id = + (SELECT department_id + FROM departments + WHERE location_id = '2100'), + (salary, commission_pct) = + (SELECT 1.1*AVG(salary), 1.5*AVG(commission_pct) + FROM employees b + WHERE a.department_id = b.department_id) + WHERE department_id IN + (SELECT department_id + FROM departments + WHERE location_id = 2900 + OR location_id = 2700); + +UPDATE sales PARTITION (sales_q1_1999) s + SET s.promo_id = 494 + WHERE amount_sold > 1000; + +UPDATE people_demo1 p SET VALUE(p) = + (SELECT VALUE(q) FROM people_demo2 q + WHERE p.department_id = q.department_id) + WHERE p.department_id = 10; + +UPDATE employees + SET job_id ='SA_MAN', salary = salary + 1000, department_id = 140 + WHERE last_name = 'Jones' + RETURNING salary*0.25, last_name, department_id + INTO :bnd1, :bnd2, :bnd3; + +UPDATE employees + SET salary = salary * 1.1 + WHERE department_id = 100 + RETURNING SUM(salary) INTO :bnd1; + +END; diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementRef.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementRef.pls new file mode 100644 index 0000000000..dfe2ee55f0 --- /dev/null +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementRef.pls @@ -0,0 +1,12 @@ +BEGIN + +UPDATE references REF + SET REF.INST_CITY = 'Munich'; + +UPDATE references REF + SET REF.INST_CITY = DEFAULT; + +UPDATE references REF + SET (REF.INST_CITY) = (SELECT 'Munich' FROM DUAL); + +END; From 984928ad521f4c8c7f375b29c8c4b41d73b8780f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 15 Feb 2019 22:25:22 +0100 Subject: [PATCH 71/90] [plsql] Add more code samples --- .../pmd/lang/plsql/ast/WhereClauseTest.java | 2 +- .../pmd/lang/plsql/ast/SelectExpressions.pls | 14 ++++++++++++++ .../pmd/lang/plsql/ast/UpdateStatementExample2.pls | 6 ++++++ .../pmd/lang/plsql/ast/WhereClauseFunctionCall.pls | 6 ++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java index a90046db17..fdc3d87002 100644 --- a/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java +++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/lang/plsql/ast/WhereClauseTest.java @@ -21,7 +21,7 @@ public class WhereClauseTest extends AbstractPLSQLParserTst { StandardCharsets.UTF_8); ASTInput input = parsePLSQL(code); List selectStatements = input.findDescendantsOfType(ASTSelectIntoStatement.class); - Assert.assertEquals(3, selectStatements.size()); + Assert.assertEquals(4, selectStatements.size()); ASTFunctionCall functionCall = selectStatements.get(0).getFirstDescendantOfType(ASTFunctionCall.class); Assert.assertEquals("UPPER", functionCall.getImage()); diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls index 1f2921da36..813c3deef4 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectExpressions.pls @@ -60,5 +60,19 @@ SELECT listagg(e.email,',') within group (order by e.email )INTO v_task_resp FROM sso_auth_employees e; +select listagg(asap_func_loc_number,'; ') within group (order by 1) + INTO my_record + FROM company_asap_func_locs + WHERE cmp_id = cmp_id_in; + +SELECT CASE + WHEN priv != 'Y' AND my_package.my_function(param1, TO_NUMBER(TO_CHAR(SYSDATE, 'yyyy'))) >= 100 THEN + 'Y' + ELSE + 'N' + END + INTO my_result + FROM DUAL; + END; / \ No newline at end of file diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls index 02d8c76ad1..272fc16109 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/UpdateStatementExample2.pls @@ -52,4 +52,10 @@ UPDATE employees WHERE department_id = 100 RETURNING SUM(salary) INTO :bnd1; +update xsearch_wsh_active + set line_items_cnt = p_cnt + where wsh_id = p_wid + and revision = p_rev + returning opp_id into v_opp_id; + END; diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseFunctionCall.pls b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseFunctionCall.pls index 484569f3ec..d6453ac78e 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseFunctionCall.pls +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/WhereClauseFunctionCall.pls @@ -21,5 +21,11 @@ BEGIN FROM mytable WHERE colname = utils.get_colname('COLUMN_ID'); + SELECT foo + INTO bar + FROM DUAL + WHERE a = 1 AND b = 2 AND NVL(INSTR(c || d), 3) = 3 + ORDER BY 1; + END; / From 22aaa70983c093d6ca5e3f1e55a57903716e1424 Mon Sep 17 00:00:00 2001 From: Matias Fraga Date: Sat, 16 Feb 2019 16:20:03 -0300 Subject: [PATCH 72/90] Add final modifier --- .../src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index b4ee895e5f..af87b2f783 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -23,7 +23,7 @@ public class KotlinTokenizer extends AntlrTokenizer { } @Override - protected AntlrTokenFilter getTokenFilter(AntlrTokenManager tokenManager) { + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { return new KotlinTokenFilter(tokenManager); } From 808c03f532cfcbbdc38a490e625e1ac91297472e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 16 Feb 2019 20:23:00 +0100 Subject: [PATCH 73/90] [plsql] Add SimpleExpressionNode, allow expressions with REGEXP_LIKE --- pmd-plsql/etc/grammar/PldocAST.jjt | 74 +++++++++++++------ .../plsql/ast/ASTRegexpLikeCondition.java | 8 +- .../plsql/ast/PLSQLParserVisitorAdapter.java | 5 ++ .../lang/plsql/rule/AbstractPLSQLRule.java | 5 ++ .../lang/plsql/ast/SelectExpressionsTest.java | 17 +++++ .../pmd/lang/plsql/ast/WhereClauseTest.java | 2 +- .../lang/plsql/ast/SelectSimpleExpression.pls | 12 +++ .../lang/plsql/ast/WhereClauseRegexpLike.pls | 10 +++ 8 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SelectSimpleExpression.pls diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 3e4f6222bd..9f59acd75e 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -1305,6 +1305,8 @@ void Condition2() #void : ( // a IS OF Type condition that starts with a function... LOOKAHEAD(FunctionCall() [] ) IsOfTypeCondition() + | + LOOKAHEAD() RegexpLikeCondition() | LOOKAHEAD(9) MultisetCondition() | @@ -1315,8 +1317,6 @@ void Condition2() #void : LOOKAHEAD(9) InCondition() | LOOKAHEAD(9) LikeCondition() - | - LOOKAHEAD(9) RegexpLikeCondition() | LOOKAHEAD(4) BetweenCondition() | @@ -1355,8 +1355,8 @@ ASTRegexpLikeCondition RegexpLikeCondition() : } { "(" - ID() "," - StringLiteral() + SqlExpression() "," + SqlExpression() [ "," ( matchParam = | matchParam = ) {jjtThis.setMatchParam(matchParam.toString());} ] ")" { return jjtThis; } @@ -1473,26 +1473,51 @@ ASTExpressionList ExpressionList() : * @see https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/About-SQL-Expressions.html */ ASTSqlExpression SqlExpression() : -{} +{ PLSQLNode simpleNode = null; } { - [ LOOKAHEAD(2) ] - // SimpleExpression ( - LOOKAHEAD(3) AdditiveExpression() // this can be a literal or a simple expression, but no conditional - | - LOOKAHEAD(5) SchemaName() "." TableName() "." Column() - | - LOOKAHEAD(3) TableName() "." Column() - | - LOOKAHEAD(2) Column() - | - LOOKAHEAD(2) - | - LOOKAHEAD(2) "(" SelectStatement() ")" // see "Scalar Subquery Expressions" + // AdditiveExpression includes literals and simple expressions, but no conditional + ( simpleNode = AdditiveExpression() ) { jjtThis.setImage(simpleNode.getImage()); } ) { return jjtThis; } } +/** + * https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Simple-Expressions.html + */ +ASTSimpleExpression SimpleExpression() : +{ StringBuilder sb = new StringBuilder(); } +{ + ( + LOOKAHEAD(2) { sb.append(token.image); } + | + StringLiteral() { sb.append(token.image); } + | + NumericLiteral() { sb.append(token.image); } + | + LOOKAHEAD(2) NullLiteral() { sb.append(token.image); } + | + // sequence + LOOKAHEAD(4) ID() "." ( | ) + | + LOOKAHEAD(6) + SchemaName() { sb.append(token.image); } "." { sb.append(token.image); } + TableName() { sb.append(token.image); } "." { sb.append(token.image); } + Column() { sb.append(token.image); } + | + LOOKAHEAD(4) + TableName() { sb.append(token.image); } "." { sb.append(token.image); } + Column() { sb.append(token.image); } + | + LOOKAHEAD(2) + Column() { sb.append(token.image); } + ) + { + jjtThis.setImage(sb.toString()); + return jjtThis; + } +} + /** * Built-in function call or a user defined function call. * See https://docs.oracle.com/en/database/oracle/oracle-database/18/sqlrf/Functions.html#GUID-D079EFD3-C683-441F-977E-2C9503089982 @@ -2938,6 +2963,7 @@ ASTUnaryExpression UnaryExpression(boolean isUnarySign) #UnaryExpression(>1) : ( ( ("+" ) { sb.append(isUnarySign ? " +" : " + "); } | ("-" ) { sb.append(isUnarySign ? " -" : " - "); } + | () { sb.append(" PRIOR "); } ) (simpleNode = UnaryExpression(false) ) { sb.append(simpleNode.getImage()); } | @@ -3039,12 +3065,13 @@ ASTPrimaryPrefix PrimaryPrefix() : ( // Note: AnalyticClause and WithinClause are only allowed for specific functions, but this grammar allows it for all functions. LOOKAHEAD(FunctionName() "(") simpleNode = FunctionCall() [ AnalyticClause() ] [ WithinClause() ] -| LOOKAHEAD(Literal()) simpleNode = Literal() { sb.append(simpleNode.getImage()) ; } | LOOKAHEAD(MultiSetCondition()) simpleNode = MultiSetCondition() | LOOKAHEAD(TrimExpression()) simpleNode = TrimExpression() //SRT 20110613.3 | LOOKAHEAD(CaseExpression()) ( simpleNode =CaseExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110520 | LOOKAHEAD(ObjectExpression() ) ( simpleNode = ObjectExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110604 //| LOOKAHEAD(LikeExpression()) ( simpleNode = LikeExpression() ) { sb.append(simpleNode.getImage()) ; } //SRT 20110604 +| LOOKAHEAD(SimpleExpression()) ( simpleNode = SimpleExpression() ) { sb.append(simpleNode.getImage()); } +| LOOKAHEAD(Literal()) ( simpleNode = Literal() ) { sb.append(simpleNode.getImage()) ; } | ( simpleNode =Name() ) { sb.append(simpleNode.getImage()) ; } | SelectStatement() | LOOKAHEAD("(" //SYNTAX //RESERVED WORD | | //RESERVED WORD | //RESERVED WORD @@ -6405,7 +6404,6 @@ ASTQualifiedID QualifiedID(): {} //20120501 | | //20120501 | - // -| | | -| +| | -| +| +| |
| | -| +| | | | | -| -| | +| +| | | -) - { jjtThis.setImage(token.toString()) ; jjtThis.value = token ; return jjtThis ; } } ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} @@ -6505,33 +6521,3 @@ ASTJavaInterfaceClass JavaInterfaceClass(): {} { jjtThis.setImage(token.toString()) ; jjtThis.value = token ; return jjtThis ; } } - - - -/** - * Tests, if the new Token if type ID is identical to the old Token of type ID. - * - * 2006-05-18 - Matthias Hendler - added: Just for understanding javacc and not used, - * cause to many closing ENDs (IDENTIFIER) were mistyped. - */ -PLSQLNode EqualsOldIDNewID(PLSQLNode pOldID) : -{ - PLSQLNode newID; - Token oldIDToken; - Token newIDToken; -} -{ - newID = ID() - { - oldIDToken = (Token) ((AbstractPLSQLNode)pOldID).value ; - newIDToken = (Token) ((AbstractPLSQLNode)newID).value ; - if (oldIDToken.image.equals(newIDToken.image)) { - return newID; - } else { - throw new ParseException("PLSQL syntax error on line "+newIDToken.beginLine+" at column "+newIDToken.beginColumn+"!\n" - + "Found \""+newIDToken.image+"\" but expected \""+oldIDToken.image+"\"."); - } - } - -} - diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java index 669b2de2d8..9ec26fdcfe 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParserVisitorAdapter.java @@ -646,11 +646,6 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { return visit((PLSQLNode) node, data); } - @Override - public Object visit(ASTKEYWORD_RESERVED node, Object data) { - return visit((PLSQLNode) node, data); - } - @Override public Object visit(ASTKEYWORD_UNRESERVED node, Object data) { return visit((PLSQLNode) node, data); @@ -681,11 +676,6 @@ public class PLSQLParserVisitorAdapter implements PLSQLParserVisitor { return visit((PLSQLNode) node, data); } - @Override - public Object visit(ASTEqualsOldIDNewID node, Object data) { - return visit((PLSQLNode) node, data); - } - @Override public Object visit(ASTAccessibleByClause node, Object data) { return visit((PLSQLNode) node, data); diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java index 9a1a272d9a..136e84b364 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/rule/AbstractPLSQLRule.java @@ -740,11 +740,6 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } - @Override - public Object visit(ASTKEYWORD_RESERVED node, Object data) { - return visit((PLSQLNode) node, data); - } - @Override public Object visit(ASTKEYWORD_UNRESERVED node, Object data) { return visit((PLSQLNode) node, data); @@ -775,11 +770,6 @@ public abstract class AbstractPLSQLRule extends AbstractRule implements PLSQLPar return visit((PLSQLNode) node, data); } - @Override - public Object visit(ASTEqualsOldIDNewID node, Object data) { - return visit((PLSQLNode) node, data); - } - @Override public Object visit(ASTAccessibleByClause node, Object data) { return visit((PLSQLNode) node, data); diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveMethodLength.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveMethodLength.xml index 3099f0b7fc..1dc8a5d163 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveMethodLength.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveMethodLength.xml @@ -247,8 +247,6 @@ IS COLUMN_STATS VARCHAR2(240); /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240); -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240); /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240); /** Unreserved Keyword Word COMMIT **/ @@ -723,8 +721,6 @@ IS INDEX_STATS VARCHAR2(240); /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240); -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPES **/ @@ -1317,8 +1313,6 @@ IS OVERFLOW VARCHAR2(240); /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240); -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240); /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING VARCHAR2(240); /** Unreserved Keyword Word OWN **/ @@ -2220,7 +2214,6 @@ BEGIN COLUMN := 'COLUMN' ; -- COLUMN VARCHAR2(240) COLUMN_STATS := 'COLUMN_STATS' ; -- COLUMN_STATS VARCHAR2(240) COLUMN_VALUE := 'COLUMN_VALUE' ; -- COLUMN_VALUE VARCHAR2(240) - COLUMNS := 'COLUMNS' ; -- COLUMNS VARCHAR2(240) COMMENT := 'COMMENT' ; -- COMMENT VARCHAR2(240) COMMIT := 'COMMIT' ; -- COMMIT VARCHAR2(240) COMMITTED := 'COMMITTED' ; -- COMMITTED VARCHAR2(240) @@ -2458,7 +2451,6 @@ BEGIN INDEX_SS_DESC := 'INDEX_SS_DESC' ; -- INDEX_SS_DESC VARCHAR2(240) INDEX_STATS := 'INDEX_STATS' ; -- INDEX_STATS VARCHAR2(240) INDEXED := 'INDEXED' ; -- INDEXED VARCHAR2(240) - INDEXES := 'INDEXES' ; -- INDEXES VARCHAR2(240) INDEXTYPE := 'INDEXTYPE' ; -- INDEXTYPE VARCHAR2(240) INDEXTYPES := 'INDEXTYPES' ; -- INDEXTYPES VARCHAR2(240) --SRT 20110508 INDICATOR := 'INDICATOR' ; -- INDICATOR VARCHAR2(240) @@ -2754,7 +2746,6 @@ BEGIN OVER := 'OVER' ; -- OVER VARCHAR2(240) OVERFLOW := 'OVERFLOW' ; -- OVERFLOW VARCHAR2(240) OVERFLOW_NOMOVE := 'OVERFLOW_NOMOVE' ; -- OVERFLOW_NOMOVE VARCHAR2(240) - OVERLAPS := 'OVERLAPS' ; -- OVERLAPS VARCHAR2(240) OVERRIDING := 'OVERRIDING' ; -- OVERRIDING VARCHAR2(240) OWN := 'OWN' ; -- OWN VARCHAR2(240) P := 'P' ; -- P VARCHAR2(240) diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveObjectLength.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveObjectLength.xml index 1223bc4c52..fdabfd9af9 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveObjectLength.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveObjectLength.xml @@ -245,8 +245,6 @@ TYPE typ_unreserved_keywords AS OBJECT COLUMN_STATS VARCHAR2(240), /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240), -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240), /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240), /** Unreserved Keyword Word COMMIT **/ @@ -721,8 +719,6 @@ TYPE typ_unreserved_keywords AS OBJECT INDEX_STATS VARCHAR2(240), /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240), -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240), /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240), /** Unreserved Keyword Word INDEXTYPES **/ @@ -1315,8 +1311,6 @@ TYPE typ_unreserved_keywords AS OBJECT OVERFLOW VARCHAR2(240), /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240), -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240), /** Unreserved Keyword Word OVERRIDING **/ -- OVERRIDING VARCHAR2(240), /** Unreserved Keyword Word OWN **/ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessivePackageSpecificationLength.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessivePackageSpecificationLength.xml index cf8202d8cc..1d5cee6736 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessivePackageSpecificationLength.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessivePackageSpecificationLength.xml @@ -245,8 +245,6 @@ PACKAGE pkg_unreserved AS COLUMN_STATS VARCHAR2(240); /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240); -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240); /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240); /** Unreserved Keyword Word COMMIT **/ @@ -721,8 +719,6 @@ PACKAGE pkg_unreserved AS INDEX_STATS VARCHAR2(240); /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240); -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPES **/ @@ -1315,8 +1311,6 @@ PACKAGE pkg_unreserved AS OVERFLOW VARCHAR2(240); /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240); -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240); /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING VARCHAR2(240); /** Unreserved Keyword Word OWN **/ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveParameterList.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveParameterList.xml index abe04868d6..6e7dc24f9b 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveParameterList.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveParameterList.xml @@ -248,8 +248,6 @@ prc_test_unreserved_keyword COLUMN_STATS IN OUT VARCHAR2, /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE IN OUT VARCHAR2, -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS IN OUT VARCHAR2, /** Unreserved Keyword Word COMMENT **/ COMMENT IN OUT VARCHAR2, /** Unreserved Keyword Word COMMIT **/ @@ -724,8 +722,6 @@ prc_test_unreserved_keyword INDEX_STATS IN OUT VARCHAR2, /** Unreserved Keyword Word INDEXED **/ INDEXED IN OUT VARCHAR2, -/** Unreserved Keyword Word INDEXES **/ - INDEXES IN OUT VARCHAR2, /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE IN OUT VARCHAR2, /** Unreserved Keyword Word INDEXTYPES **/ @@ -1318,8 +1314,6 @@ prc_test_unreserved_keyword OVERFLOW IN OUT VARCHAR2, /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE IN OUT VARCHAR2, -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS IN OUT VARCHAR2, /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING IN OUT VARCHAR2, /** Unreserved Keyword Word OWN **/ @@ -2223,7 +2217,6 @@ BEGIN COLUMN := 'COLUMN' ; -- COLUMN VARCHAR2(240) COLUMN_STATS := 'COLUMN_STATS' ; -- COLUMN_STATS VARCHAR2(240) COLUMN_VALUE := 'COLUMN_VALUE' ; -- COLUMN_VALUE VARCHAR2(240) - COLUMNS := 'COLUMNS' ; -- COLUMNS VARCHAR2(240) COMMENT := 'COMMENT' ; -- COMMENT VARCHAR2(240) COMMIT := 'COMMIT' ; -- COMMIT VARCHAR2(240) COMMITTED := 'COMMITTED' ; -- COMMITTED VARCHAR2(240) @@ -2461,7 +2454,6 @@ BEGIN INDEX_SS_DESC := 'INDEX_SS_DESC' ; -- INDEX_SS_DESC VARCHAR2(240) INDEX_STATS := 'INDEX_STATS' ; -- INDEX_STATS VARCHAR2(240) INDEXED := 'INDEXED' ; -- INDEXED VARCHAR2(240) - INDEXES := 'INDEXES' ; -- INDEXES VARCHAR2(240) INDEXTYPE := 'INDEXTYPE' ; -- INDEXTYPE VARCHAR2(240) INDEXTYPES := 'INDEXTYPES' ; -- INDEXTYPES VARCHAR2(240) --SRT 20110508 INDICATOR := 'INDICATOR' ; -- INDICATOR VARCHAR2(240) @@ -2757,7 +2749,6 @@ BEGIN OVER := 'OVER' ; -- OVER VARCHAR2(240) OVERFLOW := 'OVERFLOW' ; -- OVERFLOW VARCHAR2(240) OVERFLOW_NOMOVE := 'OVERFLOW_NOMOVE' ; -- OVERFLOW_NOMOVE VARCHAR2(240) - OVERLAPS := 'OVERLAPS' ; -- OVERLAPS VARCHAR2(240) OVERRIDING := 'OVERRIDING' ; -- OVERRIDING VARCHAR2(240) OWN := 'OWN' ; -- OWN VARCHAR2(240) P := 'P' ; -- P VARCHAR2(240) diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveTypeLength.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveTypeLength.xml index 6063779f02..3129a44cb1 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveTypeLength.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/ExcessiveTypeLength.xml @@ -246,8 +246,6 @@ TYPE typ_unreserved_keywords AS OBJECT COLUMN_STATS VARCHAR2(240), /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240), -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240), /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240), /** Unreserved Keyword Word COMMIT **/ @@ -722,8 +720,6 @@ TYPE typ_unreserved_keywords AS OBJECT INDEX_STATS VARCHAR2(240), /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240), -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240), /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240), /** Unreserved Keyword Word INDEXTYPES **/ @@ -1316,8 +1312,6 @@ TYPE typ_unreserved_keywords AS OBJECT OVERFLOW VARCHAR2(240), /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240), -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240), /** Unreserved Keyword Word OVERRIDING **/ -- OVERRIDING VARCHAR2(240), /** Unreserved Keyword Word OWN **/ diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssMethodCount.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssMethodCount.xml index 0a57886649..896ed43abe 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssMethodCount.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssMethodCount.xml @@ -265,8 +265,6 @@ AS COLUMN_STATS VARCHAR2(240); /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240); -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240); /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240); /** Unreserved Keyword Word COMMIT **/ @@ -741,8 +739,6 @@ AS INDEX_STATS VARCHAR2(240); /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240); -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPES **/ @@ -1335,8 +1331,6 @@ AS OVERFLOW VARCHAR2(240); /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240); -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240); /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING VARCHAR2(240); /** Unreserved Keyword Word OWN **/ @@ -2238,7 +2232,6 @@ BEGIN COLUMN := 'COLUMN' ; -- COLUMN VARCHAR2(240) COLUMN_STATS := 'COLUMN_STATS' ; -- COLUMN_STATS VARCHAR2(240) COLUMN_VALUE := 'COLUMN_VALUE' ; -- COLUMN_VALUE VARCHAR2(240) - COLUMNS := 'COLUMNS' ; -- COLUMNS VARCHAR2(240) COMMENT := 'COMMENT' ; -- COMMENT VARCHAR2(240) COMMIT := 'COMMIT' ; -- COMMIT VARCHAR2(240) COMMITTED := 'COMMITTED' ; -- COMMITTED VARCHAR2(240) @@ -2476,7 +2469,6 @@ BEGIN INDEX_SS_DESC := 'INDEX_SS_DESC' ; -- INDEX_SS_DESC VARCHAR2(240) INDEX_STATS := 'INDEX_STATS' ; -- INDEX_STATS VARCHAR2(240) INDEXED := 'INDEXED' ; -- INDEXED VARCHAR2(240) - INDEXES := 'INDEXES' ; -- INDEXES VARCHAR2(240) INDEXTYPE := 'INDEXTYPE' ; -- INDEXTYPE VARCHAR2(240) INDEXTYPES := 'INDEXTYPES' ; -- INDEXTYPES VARCHAR2(240) --SRT 20110508 INDICATOR := 'INDICATOR' ; -- INDICATOR VARCHAR2(240) @@ -2772,7 +2764,6 @@ BEGIN OVER := 'OVER' ; -- OVER VARCHAR2(240) OVERFLOW := 'OVERFLOW' ; -- OVERFLOW VARCHAR2(240) OVERFLOW_NOMOVE := 'OVERFLOW_NOMOVE' ; -- OVERFLOW_NOMOVE VARCHAR2(240) - OVERLAPS := 'OVERLAPS' ; -- OVERLAPS VARCHAR2(240) OVERRIDING := 'OVERRIDING' ; -- OVERRIDING VARCHAR2(240) OWN := 'OWN' ; -- OWN VARCHAR2(240) P := 'P' ; -- P VARCHAR2(240) diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssObjectCount.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssObjectCount.xml index 1f02d88959..5987fe9dd3 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssObjectCount.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/NcssObjectCount.xml @@ -249,8 +249,6 @@ AS COLUMN_STATS VARCHAR2(240); /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240); -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240); /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240); /** Unreserved Keyword Word COMMIT **/ @@ -725,8 +723,6 @@ AS INDEX_STATS VARCHAR2(240); /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240); -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPES **/ @@ -1319,8 +1315,6 @@ AS OVERFLOW VARCHAR2(240); /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240); -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240); /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING VARCHAR2(240); /** Unreserved Keyword Word OWN **/ @@ -2228,7 +2222,6 @@ BEGIN COLUMN := 'COLUMN' ; -- COLUMN VARCHAR2(240) COLUMN_STATS := 'COLUMN_STATS' ; -- COLUMN_STATS VARCHAR2(240) COLUMN_VALUE := 'COLUMN_VALUE' ; -- COLUMN_VALUE VARCHAR2(240) - COLUMNS := 'COLUMNS' ; -- COLUMNS VARCHAR2(240) COMMENT := 'COMMENT' ; -- COMMENT VARCHAR2(240) COMMIT := 'COMMIT' ; -- COMMIT VARCHAR2(240) COMMITTED := 'COMMITTED' ; -- COMMITTED VARCHAR2(240) @@ -2466,7 +2459,6 @@ BEGIN INDEX_SS_DESC := 'INDEX_SS_DESC' ; -- INDEX_SS_DESC VARCHAR2(240) INDEX_STATS := 'INDEX_STATS' ; -- INDEX_STATS VARCHAR2(240) INDEXED := 'INDEXED' ; -- INDEXED VARCHAR2(240) - INDEXES := 'INDEXES' ; -- INDEXES VARCHAR2(240) INDEXTYPE := 'INDEXTYPE' ; -- INDEXTYPE VARCHAR2(240) INDEXTYPES := 'INDEXTYPES' ; -- INDEXTYPES VARCHAR2(240) --SRT 20110508 INDICATOR := 'INDICATOR' ; -- INDICATOR VARCHAR2(240) @@ -2768,7 +2760,6 @@ BEGIN OVER := 'OVER' ; -- OVER VARCHAR2(240) OVERFLOW := 'OVERFLOW' ; -- OVERFLOW VARCHAR2(240) OVERFLOW_NOMOVE := 'OVERFLOW_NOMOVE' ; -- OVERFLOW_NOMOVE VARCHAR2(240) - OVERLAPS := 'OVERLAPS' ; -- OVERLAPS VARCHAR2(240) OVERRIDING := 'OVERRIDING' ; -- OVERRIDING VARCHAR2(240) OWN := 'OWN' ; -- OWN VARCHAR2(240) P := 'P' ; -- P VARCHAR2(240) diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/TooManyFields.xml b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/TooManyFields.xml index d9453eb8aa..2fa4e9737c 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/TooManyFields.xml +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/rule/design/xml/TooManyFields.xml @@ -245,8 +245,6 @@ AS COLUMN_STATS VARCHAR2(240); /** Unreserved Keyword Word COLUMN_VALUE **/ COLUMN_VALUE VARCHAR2(240); -/** Unreserved Keyword Word COLUMNS **/ - COLUMNS VARCHAR2(240); /** Unreserved Keyword Word COMMENT **/ COMMENT VARCHAR2(240); /** Unreserved Keyword Word COMMIT **/ @@ -721,8 +719,6 @@ AS INDEX_STATS VARCHAR2(240); /** Unreserved Keyword Word INDEXED **/ INDEXED VARCHAR2(240); -/** Unreserved Keyword Word INDEXES **/ - INDEXES VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPE **/ INDEXTYPE VARCHAR2(240); /** Unreserved Keyword Word INDEXTYPES **/ @@ -1315,8 +1311,6 @@ AS OVERFLOW VARCHAR2(240); /** Unreserved Keyword Word OVERFLOW_NOMOVE **/ OVERFLOW_NOMOVE VARCHAR2(240); -/** Unreserved Keyword Word OVERLAPS **/ - OVERLAPS VARCHAR2(240); /** Unreserved Keyword Word OVERRIDING **/ OVERRIDING VARCHAR2(240); /** Unreserved Keyword Word OWN **/ From c32daec6992222455fa2e7c5367d2c1e5c8734b4 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 16 Feb 2019 19:23:39 +0100 Subject: [PATCH 79/90] [plsql] Avoiding tokens for "FIRST" and "REPLACE" --- pmd-plsql/etc/grammar/PldocAST.jjt | 40 ++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt index 819549733c..87715b4055 100644 --- a/pmd-plsql/etc/grammar/PldocAST.jjt +++ b/pmd-plsql/etc/grammar/PldocAST.jjt @@ -358,7 +358,7 @@ ASTPackageSpecification PackageSpecification() : } { ( - [ [ "REPLACE"] [ | ] ] + [ [ KEYWORD("REPLACE")] [ | ] ] simpleNode = ObjectNameDeclaration() ( @@ -389,7 +389,7 @@ ASTPackageBody PackageBody() : } { ( - [ [ "REPLACE"] [ | ] ] + [ [ KEYWORD("REPLACE")] [ | ] ] ( | ) simpleNode = ObjectNameDeclaration() ( @@ -491,7 +491,7 @@ ASTProgramUnit ProgramUnit() : { ( - [ [ "REPLACE"] [ | ] ] + [ [ KEYWORD("REPLACE")] [ | ] ] MethodDeclarator() @@ -1195,7 +1195,7 @@ void OrderByEntry() #void : { SqlExpression() [ | ] - [ ( "FIRST" | "LAST" ) ] + [ ( KEYWORD("FIRST") | "LAST" ) ] } /** @@ -1207,7 +1207,7 @@ ASTRowLimitingClause RowLimitingClause() : ( NumericLiteral() ( | ) | - ( "FIRST" | ) [ NumericLiteral() [ ] ] ( | ) ( | ) + ( KEYWORD("FIRST") | ) [ NumericLiteral() [ ] ] ( | ) ( | ) ) { return jjtThis; } } @@ -2147,7 +2147,7 @@ ASTMultiTableInsert MultiTableInsert() : ASTConditionalInsertClause ConditionalInsertClause() : {} { - [ | "FIRST" ] ( Condition() ( InsertIntoClause() [ ValuesClause() ] )+ )+ + [ | KEYWORD("FIRST") ] ( Condition() ( InsertIntoClause() [ ValuesClause() ] )+ )+ [ ( InsertIntoClause() [ ValuesClause() ] )+ ] { return jjtThis; } } @@ -3524,7 +3524,7 @@ ASTView View() : PLSQLNode simpleNode = null; } { - [ "REPLACE"] + [ KEYWORD("REPLACE")] [[] ] simpleNode = ObjectNameDeclaration() ["(" ViewColumn() ("," ViewColumn())* ")"] @@ -3541,7 +3541,7 @@ ASTSynonym Synonym() : PLSQLNode simpleNode = null; } { - [ "REPLACE"] + [ KEYWORD("REPLACE")] [] simpleNode = ObjectNameDeclaration() @@ -3556,7 +3556,7 @@ ASTDirectory Directory() : PLSQLNode simpleNode = null; } { - [ "REPLACE"] + [ KEYWORD("REPLACE")] simpleNode = ObjectNameDeclaration() @@ -3673,7 +3673,7 @@ ASTTypeSpecification TypeSpecification() : PLSQLNode simpleNode = null; } { - [ [ "REPLACE"] [ | ] ] + [ [ KEYWORD("REPLACE")] [ | ] ] simpleNode = ObjectNameDeclaration() [ @@ -3852,7 +3852,7 @@ ASTAlterTypeSpec AlterTypeSpec() : | */ [ - "REPLACE" + KEYWORD("REPLACE") ( LOOKAHEAD(2) // ( | ) ( @@ -4064,7 +4064,7 @@ ASTTriggerUnit TriggerUnit() : PLSQLNode simpleNode = null ; } { - [ [ "REPLACE"] [ | ] ] + [ [ KEYWORD("REPLACE")] [ | ] ] () simpleNode = ObjectNameDeclaration() @@ -4938,11 +4938,25 @@ void RESERVED_WORD() #void: {} | } +void KEYWORD(String id) #void: +{} +{ + + { + if (!token.image.equalsIgnoreCase(id)) { + String eol = System.getProperty("line.separator", "\n"); + throw new ParseException("Encountered \"" + token.image + "\" at line " + + token.beginLine + ", column " + token.beginColumn + "." + eol + + "Was expecting: " + id); + } + } +} + ASTKEYWORD_UNRESERVED KEYWORD_UNRESERVED (): {} { // PL/SQL UNRESERVED KEYWORDS - V$RESERVED.RESERVED='N' ( -"FIRST" | "REPLACE" | "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | "WITHIN" | + "REF" | "LAST" | "TRIM" | "OVER" | "UNBOUNDED" | "PRECEDING" | "FOLLOWING" | "WITHIN" | "OVERFLOW" | "ERROR" | "WITHOUT" | "COUNT" | "VALUE" | "SUBPARTITION" | "LOG" | "ERRORS" | "REJECT" | "UNLIMITED" | | From f3771d2401fcd8c9ce5c13448133b4f2cc1e6def Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 16 Feb 2019 22:17:23 +0100 Subject: [PATCH 80/90] Update release notes, fixes #1656 --- docs/pages/release_notes.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index f7a1fc64c7..301ded665a 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,13 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### PL/SQL Grammar improvements + +* In this release, many more parser bugs in our PL/SQL support have been fixed. This adds more complete + support for UPDATE statements and subqueries and hierarchical queries in SELECT statements. +* Support for analytic functions such as LISTAGG has been added. +* Conditions in WHERE clauses support now REGEX_LIKE and multiset conditions. + #### Modified Rules * The Apex rule {% rule "apex/codestyle/MethodNamingConventions" %} (apex-codestyle) has a new @@ -38,9 +45,10 @@ This is a {{ site.pmd.release_type }} release. * java-performance * [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch * plsql - * [#1587](https://github.com/pmd/pmd/issues/1587): \[plsql] Parse Exception with EXISTS + * [#1587](https://github.com/pmd/pmd/issues/1587): \[plsql] ParseException with EXISTS * [#1589](https://github.com/pmd/pmd/issues/1589): \[plsql] ParseException with subqueries in WHERE clause * [#1590](https://github.com/pmd/pmd/issues/1590): \[plsql] ParseException when using hierarchical query clause + * [#1656](https://github.com/pmd/pmd/issues/1656): \[plsql] ParseException with analytic functions, trim and subqueries ### API Changes From 4938fa82728e395aa9bf8b4f133cd159ad2b75aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 14:33:41 -0300 Subject: [PATCH 81/90] The rule will be part of 6.12.0 --- pmd-java/src/main/resources/category/java/bestpractices.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index 03285b22f6..24fc4666fc 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -1522,7 +1522,7 @@ public class Foo { Date: Sun, 17 Feb 2019 14:43:32 -0300 Subject: [PATCH 82/90] [core] Add release sourceset --- .../src/main/resources/rulesets/releases/6120.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pmd-core/src/main/resources/rulesets/releases/6120.xml diff --git a/pmd-core/src/main/resources/rulesets/releases/6120.xml b/pmd-core/src/main/resources/rulesets/releases/6120.xml new file mode 100644 index 0000000000..873a0d1088 --- /dev/null +++ b/pmd-core/src/main/resources/rulesets/releases/6120.xml @@ -0,0 +1,14 @@ + + + + +This ruleset contains links to rules that are new in PMD v6.12.0 + + + + + + From dd175ff4e56c17937d5c74272fe3c46cb13b7351 Mon Sep 17 00:00:00 2001 From: "Travis CI (pmd-bot)" Date: Sun, 17 Feb 2019 17:58:25 +0000 Subject: [PATCH 83/90] Update documentation TRAVIS_JOB_NUMBER=3448.1 TRAVIS_COMMIT_RANGE=4835b29f88f0...9f4341e70e02 --- docs/pages/pmd/rules/java.md | 1 + docs/pages/pmd/rules/java/bestpractices.md | 65 +++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/pages/pmd/rules/java.md b/docs/pages/pmd/rules/java.md index 925c73779a..eea3a593fd 100644 --- a/docs/pages/pmd/rules/java.md +++ b/docs/pages/pmd/rules/java.md @@ -56,6 +56,7 @@ folder: pmd/rules * [UseAssertSameInsteadOfAssertTrue](pmd_rules_java_bestpractices.html#useassertsameinsteadofasserttrue): This rule detects JUnit assertions in object references equality. These assertions should be made... * [UseAssertTrueInsteadOfAssertEquals](pmd_rules_java_bestpractices.html#useasserttrueinsteadofassertequals): When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, ins... * [UseCollectionIsEmpty](pmd_rules_java_bestpractices.html#usecollectionisempty): The isEmpty() method on java.util.Collection is provided to determine if a collection has any ele... +* [UseTryWithResources](pmd_rules_java_bestpractices.html#usetrywithresources): Java 7 introduced the try-with-resources statement. This statement ensures that each resource is ... * [UseVarargs](pmd_rules_java_bestpractices.html#usevarargs): Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic... ## Code Style diff --git a/docs/pages/pmd/rules/java/bestpractices.md b/docs/pages/pmd/rules/java/bestpractices.md index e50f840abf..dba2d0ac36 100644 --- a/docs/pages/pmd/rules/java/bestpractices.md +++ b/docs/pages/pmd/rules/java/bestpractices.md @@ -5,7 +5,7 @@ permalink: pmd_rules_java_bestpractices.html folder: pmd/rules/java sidebaractiveurl: /pmd_rules_java.html editmepath: ../pmd-java/src/main/resources/category/java/bestpractices.xml -keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseVarargs +keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseTryWithResources, UseVarargs language: Java --- @@ -1636,6 +1636,69 @@ public class Foo { ``` +## UseTryWithResources + +**Since:** PMD 6.12.0 + +**Priority:** Medium (3) + +**Minimum Language Version:** Java 1.7 + +Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end +of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions +are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from +the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is +preserved. + +**This rule is defined by the following XPath expression:** +``` xpath +//TryStatement[FinallyStatement//Name[ + tokenize(@Image, '\.')[last()] = $closeMethods +][ + pmd-java:typeIs('java.lang.AutoCloseable') + or + ../../PrimarySuffix/Arguments[@ArgumentCount = 1]//PrimaryPrefix[pmd-java:typeIs('java.lang.AutoCloseable')] +]] +``` + +**Example(s):** + +``` java +public class TryWithResources { + public void run() { + InputStream in = null; + try { + in = openInputStream(); + int i = in.read(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (in != null) in.close(); + } catch (IOException ignored) { + // ignored + } + } + + // better use try-with-resources + try (InputStream in2 = openInputStream()) { + int i = in2.read(); + } + } +} +``` + +**This rule has the following properties:** + +|Name|Default Value|Description|Multivalued| +|----|-------------|-----------|-----------| +|closeMethods|close , closeQuietly|Method names in finally block, which trigger this rule|yes. Delimiter is ','.| + +**Use this rule by referencing it:** +``` xml + +``` + ## UseVarargs **Since:** PMD 5.0 From 2cf10b9759746f3ed6061b8f31832891bfaa1db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 16:31:56 -0300 Subject: [PATCH 84/90] Internalize the AntrlTokenizer --- .../main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java index d1655da3a1..fb947517cf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java @@ -8,13 +8,18 @@ import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.Token; +import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.cpd.token.AntlrToken; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. + * + * @deprecated This is an internal API. */ +@Deprecated +@InternalApi public abstract class AntlrTokenizer implements Tokenizer { protected abstract AntlrTokenManager getLexerForSource(SourceCode sourceCode); From decb9694919f6e739e74133de1e0422ef71f5465 Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Sun, 17 Feb 2019 17:36:15 -0300 Subject: [PATCH 85/90] Antlr token manager now is responsible of filtering non default tokens --- .../sourceforge/pmd/cpd/token/AntlrToken.java | 4 ++++ .../pmd/cpd/token/AntlrTokenFilter.java | 16 --------------- .../pmd/cpd/token/BaseTokenFilter.java | 20 ++++++++++--------- .../pmd/cpd/token/JavaCCTokenFilter.java | 5 ----- .../pmd/lang/antlr/AntlrTokenManager.java | 8 ++++++++ .../net/sourceforge/pmd/cpd/hello.go | 4 +++- 6 files changed, 26 insertions(+), 31 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java index 6df0fd1e50..476577e325 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java @@ -71,4 +71,8 @@ public class AntlrToken implements GenericToken { public boolean isHidden() { return token.getChannel() == Lexer.HIDDEN; } + + public boolean isDefault() { + return token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL; + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java index e1eee7b77b..f0f6d71e27 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -14,8 +14,6 @@ import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; */ public class AntlrTokenFilter extends BaseTokenFilter { - private boolean discardingHiddenTokens = false; - /** * Creates a new AntlrTokenFilter * @param tokenManager The token manager from which to retrieve tokens to be filtered @@ -28,18 +26,4 @@ public class AntlrTokenFilter extends BaseTokenFilter { protected boolean shouldStopProcessing(final AntlrToken currentToken) { return currentToken.getType() == EOF; } - - @Override - protected void analyzeToken(final AntlrToken currentToken) { - analyzeHiddenTokens(currentToken); - } - - @Override - protected boolean isLanguageSpecificDiscarding() { - return super.isLanguageSpecificDiscarding() || discardingHiddenTokens; - } - - private void analyzeHiddenTokens(final AntlrToken token) { - discardingHiddenTokens = token.isHidden(); - } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java index 4be453b919..c5451fa9ed 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java @@ -61,6 +61,17 @@ public abstract class BaseTokenFilter implements TokenFi } } + /** + * Extension point for subclasses to analyze all tokens (before filtering) + * and update internal status to decide on custom discard rules. + * + * @param currentToken The token to be analyzed + * @see #isLanguageSpecificDiscarding() + */ + protected void analyzeToken(final GenericToken currentToken) { + // noop + } + /** * Extension point for subclasses to indicate tokens are to be filtered. * @@ -78,13 +89,4 @@ public abstract class BaseTokenFilter implements TokenFi */ protected abstract boolean shouldStopProcessing(T currentToken); - /** - * Extension point for subclasses to analyze all tokens (before filtering) - * and update internal status to decide on custom discard rules. - * - * @param currentToken The token to be analyzed - * @see #isLanguageSpecificDiscarding() - */ - protected abstract void analyzeToken(T currentToken); - } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java index 3231e353e2..734b9fcff3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java @@ -25,9 +25,4 @@ public class JavaCCTokenFilter extends BaseTokenFilter { protected boolean shouldStopProcessing(final GenericToken currentToken) { return currentToken.getImage().isEmpty(); } - - @Override - protected void analyzeToken(final GenericToken currentToken) { - // noop - } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java index 0bf95b3336..48434eca01 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java @@ -34,6 +34,14 @@ public class AntlrTokenManager implements TokenManager { @Override public Object getNextToken() { + AntlrToken nextToken = getNextTokenFromAnyChannel(); + while (!nextToken.isDefault()) { + nextToken = getNextTokenFromAnyChannel(); + } + return nextToken; + } + + private AntlrToken getNextTokenFromAnyChannel() { final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null; final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment); previousToken = currentToken; diff --git a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go index 302c5a580a..a613af4286 100644 --- a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go +++ b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go @@ -26,4 +26,6 @@ import ( func main() { fmt.Println(stringutil.Reverse("!selpmaxe oG ,olleH")) -} \ No newline at end of file +} + +/* Comment */ \ No newline at end of file From 60aa1b5c2bd21de9ffad707bf0e7819b4e8cfe38 Mon Sep 17 00:00:00 2001 From: Tomi De Lucca Date: Sun, 17 Feb 2019 18:36:55 -0300 Subject: [PATCH 86/90] Moved base token filter to internal package --- .../java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java | 1 + .../java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java | 1 + .../pmd/cpd/token/{ => internal}/BaseTokenFilter.java | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) rename pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/{ => internal}/BaseTokenFilter.java (96%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java index f0f6d71e27..c76332d69e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd.token; import static org.antlr.v4.runtime.Token.EOF; +import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter; import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java index 734b9fcff3..b8feec5f64 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.cpd.token; +import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java similarity index 96% rename from pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java rename to pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index c5451fa9ed..0f43a305fb 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -2,8 +2,9 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd.token; +package net.sourceforge.pmd.cpd.token.internal; +import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; From 4199064c2c90d47386fd66bfad09ed628f983d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 18:50:32 -0300 Subject: [PATCH 87/90] Update changelog, refs #1654 --- docs/pages/release_notes.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index c5a96b48a0..786a3eb383 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,24 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +### CPD Suppression for Antlr-based languages + +[ITBA](https://www.itba.edu.ar/) students [Matías Fraga](https://github.com/matifraga), +[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini) +keep working on bringing full Antlr support to PMD. For this release, they have implemented +token filtering in an equivalent way as we did for JavaCC languages, adding support for CPD +suppressions through `CPD-OFF` and `CPD-ON` comments for all Antlr-based languages. + +This means, you can now ignore arbitrary blocks of code on: +* Go +* Kotlin +* Swift + +Simply start the suppression with any comment (single or multiline) containing `CPD-OFF`, +and resume again with a comment containing `CPD-ON`. + +More information is available in [the user documentation](pmd_userdocs_cpd.html#suppression). + #### New Rules * The new Java rule {% rule "java/bestpractices/UseTryWithResources" %) (`java-bestpractices`) searches @@ -61,6 +79,7 @@ This is a {{ site.pmd.release_type }} release. * [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly) * [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) * [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17) +* [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca) {% endtocmaker %} From 4a0bb348b646f6947f26c2a4b4b7a7c3b115f3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 18:52:05 -0300 Subject: [PATCH 88/90] Update docs, reflect changes to CPD suppression --- docs/pages/pmd/userdocs/cpd.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages/pmd/userdocs/cpd.md b/docs/pages/pmd/userdocs/cpd.md index 0faf48e738..ee3e842ea1 100644 --- a/docs/pages/pmd/userdocs/cpd.md +++ b/docs/pages/pmd/userdocs/cpd.md @@ -359,8 +359,8 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package: ## Suppression -Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Javascript**, **Matlab**, -**Objective-C**, **PL/SQL** and **Python** by including the keywords `CPD-OFF` and `CPD-ON`. +Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**, +**Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`. ```java public Object someParameterizedFactoryMethod(int x) throws Exception { From 55c2c00e7aceb4f4794d3e148fb1580c0e982e58 Mon Sep 17 00:00:00 2001 From: lsoncini Date: Sun, 17 Feb 2019 20:28:57 -0300 Subject: [PATCH 89/90] merge fixes --- .../sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java | 2 +- .../src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 0f43a305fb..747619b5f3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -69,7 +69,7 @@ public abstract class BaseTokenFilter implements TokenFi * @param currentToken The token to be analyzed * @see #isLanguageSpecificDiscarding() */ - protected void analyzeToken(final GenericToken currentToken) { + protected void analyzeToken(final T currentToken) { // noop } diff --git a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java index b4ee895e5f..5ce23f13c4 100644 --- a/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java +++ b/pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java @@ -45,7 +45,6 @@ public class KotlinTokenizer extends AntlrTokenizer { @Override protected void analyzeToken(final AntlrToken currentToken) { - super.analyzeToken(currentToken); skipPackageAndImport(currentToken); skipNewLines(currentToken); } @@ -65,7 +64,7 @@ public class KotlinTokenizer extends AntlrTokenizer { @Override protected boolean isLanguageSpecificDiscarding() { - return discardingPackageAndImport || discardingNL || super.isLanguageSpecificDiscarding(); + return discardingPackageAndImport || discardingNL; } } } From 52c4d6ed374f2bc5043e94087526ae0a9780cb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Sun, 17 Feb 2019 23:34:15 -0300 Subject: [PATCH 90/90] Update changelog, refs #1655 --- docs/pages/release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 786a3eb383..cd99b5f9a9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -80,6 +80,7 @@ More information is available in [the user documentation](pmd_userdocs_cpd.html# * [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17) * [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17) * [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca) +* [#1655](https://github.com/pmd/pmd/pull/1655): \[kotlin] Kotlin tokenizer refactor - [Lucas Soncini](https://github.com/lsoncini) {% endtocmaker %}