diff --git a/docs/pages/pmd/rules/java.md b/docs/pages/pmd/rules/java.md
index 925c73779a..eea3a593fd 100644
--- a/docs/pages/pmd/rules/java.md
+++ b/docs/pages/pmd/rules/java.md
@@ -56,6 +56,7 @@ folder: pmd/rules
* [UseAssertSameInsteadOfAssertTrue](pmd_rules_java_bestpractices.html#useassertsameinsteadofasserttrue): This rule detects JUnit assertions in object references equality. These assertions should be made...
* [UseAssertTrueInsteadOfAssertEquals](pmd_rules_java_bestpractices.html#useasserttrueinsteadofassertequals): When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, ins...
* [UseCollectionIsEmpty](pmd_rules_java_bestpractices.html#usecollectionisempty): The isEmpty() method on java.util.Collection is provided to determine if a collection has any ele...
+* [UseTryWithResources](pmd_rules_java_bestpractices.html#usetrywithresources): Java 7 introduced the try-with-resources statement. This statement ensures that each resource is ...
* [UseVarargs](pmd_rules_java_bestpractices.html#usevarargs): Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic...
## Code Style
diff --git a/docs/pages/pmd/rules/java/bestpractices.md b/docs/pages/pmd/rules/java/bestpractices.md
index e50f840abf..dba2d0ac36 100644
--- a/docs/pages/pmd/rules/java/bestpractices.md
+++ b/docs/pages/pmd/rules/java/bestpractices.md
@@ -5,7 +5,7 @@ permalink: pmd_rules_java_bestpractices.html
folder: pmd/rules/java
sidebaractiveurl: /pmd_rules_java.html
editmepath: ../pmd-java/src/main/resources/category/java/bestpractices.xml
-keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseVarargs
+keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseTryWithResources, UseVarargs
language: Java
---
@@ -1636,6 +1636,69 @@ public class Foo {
```
+## UseTryWithResources
+
+**Since:** PMD 6.12.0
+
+**Priority:** Medium (3)
+
+**Minimum Language Version:** Java 1.7
+
+Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end
+of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions
+are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from
+the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is
+preserved.
+
+**This rule is defined by the following XPath expression:**
+``` xpath
+//TryStatement[FinallyStatement//Name[
+ tokenize(@Image, '\.')[last()] = $closeMethods
+][
+ pmd-java:typeIs('java.lang.AutoCloseable')
+ or
+ ../../PrimarySuffix/Arguments[@ArgumentCount = 1]//PrimaryPrefix[pmd-java:typeIs('java.lang.AutoCloseable')]
+]]
+```
+
+**Example(s):**
+
+``` java
+public class TryWithResources {
+ public void run() {
+ InputStream in = null;
+ try {
+ in = openInputStream();
+ int i = in.read();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ if (in != null) in.close();
+ } catch (IOException ignored) {
+ // ignored
+ }
+ }
+
+ // better use try-with-resources
+ try (InputStream in2 = openInputStream()) {
+ int i = in2.read();
+ }
+ }
+}
+```
+
+**This rule has the following properties:**
+
+|Name|Default Value|Description|Multivalued|
+|----|-------------|-----------|-----------|
+|closeMethods|close , closeQuietly|Method names in finally block, which trigger this rule|yes. Delimiter is ','.|
+
+**Use this rule by referencing it:**
+``` xml
+
+```
+
## UseVarargs
**Since:** PMD 5.0
diff --git a/docs/pages/pmd/userdocs/cpd.md b/docs/pages/pmd/userdocs/cpd.md
index 0faf48e738..ee3e842ea1 100644
--- a/docs/pages/pmd/userdocs/cpd.md
+++ b/docs/pages/pmd/userdocs/cpd.md
@@ -359,8 +359,8 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package:
## Suppression
-Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Javascript**, **Matlab**,
-**Objective-C**, **PL/SQL** and **Python** by including the keywords `CPD-OFF` and `CPD-ON`.
+Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**,
+**Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`.
```java
public Object someParameterizedFactoryMethod(int x) throws Exception {
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index ee410dcd04..786a3eb383 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -14,6 +14,30 @@ This is a {{ site.pmd.release_type }} release.
### New and noteworthy
+### CPD Suppression for Antlr-based languages
+
+[ITBA](https://www.itba.edu.ar/) students [MatÃas Fraga](https://github.com/matifraga),
+[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini)
+keep working on bringing full Antlr support to PMD. For this release, they have implemented
+token filtering in an equivalent way as we did for JavaCC languages, adding support for CPD
+suppressions through `CPD-OFF` and `CPD-ON` comments for all Antlr-based languages.
+
+This means, you can now ignore arbitrary blocks of code on:
+* Go
+* Kotlin
+* Swift
+
+Simply start the suppression with any comment (single or multiline) containing `CPD-OFF`,
+and resume again with a comment containing `CPD-ON`.
+
+More information is available in [the user documentation](pmd_userdocs_cpd.html#suppression).
+
+#### New Rules
+
+* The new Java rule {% rule "java/bestpractices/UseTryWithResources" %) (`java-bestpractices`) searches
+ for try-blocks, that could be changed to a try-with-resources statement. This statement ensures that
+ each resource is closed at the end of the statement and is available since Java 7.
+
#### Modified Rules
* The Apex rule {% rule "apex/codestyle/MethodNamingConventions" %} (apex-codestyle) has a new
@@ -22,8 +46,11 @@ This is a {{ site.pmd.release_type }} release.
### Fixed Issues
+* all
+ * [#1559](https://github.com/pmd/pmd/issues/1559): \[core] CPD: Lexical error in file (no file name provided)
* java-bestpractices
* [#808](https://github.com/pmd/pmd/issues/808): \[java] AccessorMethodGeneration false positives with compile time constants
+ * [#1405](https://github.com/pmd/pmd/issues/1405): \[java] New Rule: UseTryWithResources - Replace close and IOUtils.closeQuietly with try-with-resources
* [#1555](https://github.com/pmd/pmd/issues/1555): \[java] UnusedImports false positive for method parameter type in @see Javadoc
* java-codestyle
* [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods
@@ -52,6 +79,7 @@ This is a {{ site.pmd.release_type }} release.
* [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly)
* [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17)
* [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17)
+* [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca)
{% endtocmaker %}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
index 2b0d2c56ea..46097b54fb 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
@@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
+import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
@@ -14,7 +15,11 @@ import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Generic implementation of a {@link Tokenizer} useful to any Antlr grammar.
+ *
+ * @deprecated This is an internal API.
*/
+@Deprecated
+@InternalApi
public abstract class AntlrTokenizer implements Tokenizer {
protected abstract AntlrTokenManager getLexerForSource(SourceCode sourceCode);
@@ -23,6 +28,8 @@ public abstract class AntlrTokenizer implements Tokenizer {
public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) {
final AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
+ tokenManager.setFileName(sourceCode.getFileName());
+
final AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager);
try {
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
new file mode 100644
index 0000000000..03adab05c5
--- /dev/null
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
@@ -0,0 +1,46 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd.internal;
+
+import java.io.IOException;
+
+import net.sourceforge.pmd.cpd.SourceCode;
+import net.sourceforge.pmd.cpd.TokenEntry;
+import net.sourceforge.pmd.cpd.Tokenizer;
+import net.sourceforge.pmd.cpd.Tokens;
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
+
+public abstract class JavaCCTokenizer implements Tokenizer {
+
+ protected abstract TokenManager getLexerForSource(SourceCode sourceCode);
+
+ protected TokenFilter getTokenFilter(TokenManager tokenManager) {
+ return new JavaCCTokenFilter(tokenManager);
+ }
+
+ protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) {
+ return new TokenEntry(currentToken.getImage(), filename, currentToken.getBeginLine());
+ }
+
+ @Override
+ public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
+ TokenManager tokenManager = getLexerForSource(sourceCode);
+ tokenManager.setFileName(sourceCode.getFileName());
+ try {
+ final TokenFilter tokenFilter = getTokenFilter(tokenManager);
+
+ GenericToken currentToken = tokenFilter.getNextToken();
+ while (currentToken != null) {
+ tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName()));
+ currentToken = tokenFilter.getNextToken();
+ }
+ } finally {
+ tokenEntries.add(TokenEntry.getEOF());
+ }
+ }
+}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java
index 6df0fd1e50..476577e325 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java
@@ -71,4 +71,8 @@ public class AntlrToken implements GenericToken {
public boolean isHidden() {
return token.getChannel() == Lexer.HIDDEN;
}
+
+ public boolean isDefault() {
+ return token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL;
+ }
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java
index e1eee7b77b..c76332d69e 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java
@@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd.token;
import static org.antlr.v4.runtime.Token.EOF;
+import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
/**
@@ -14,8 +15,6 @@ import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
*/
public class AntlrTokenFilter extends BaseTokenFilter {
- private boolean discardingHiddenTokens = false;
-
/**
* Creates a new AntlrTokenFilter
* @param tokenManager The token manager from which to retrieve tokens to be filtered
@@ -28,18 +27,4 @@ public class AntlrTokenFilter extends BaseTokenFilter {
protected boolean shouldStopProcessing(final AntlrToken currentToken) {
return currentToken.getType() == EOF;
}
-
- @Override
- protected void analyzeToken(final AntlrToken currentToken) {
- analyzeHiddenTokens(currentToken);
- }
-
- @Override
- protected boolean isLanguageSpecificDiscarding() {
- return super.isLanguageSpecificDiscarding() || discardingHiddenTokens;
- }
-
- private void analyzeHiddenTokens(final AntlrToken token) {
- discardingHiddenTokens = token.isHidden();
- }
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
index 3231e353e2..b8feec5f64 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
@@ -4,6 +4,7 @@
package net.sourceforge.pmd.cpd.token;
+import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@@ -25,9 +26,4 @@ public class JavaCCTokenFilter extends BaseTokenFilter {
protected boolean shouldStopProcessing(final GenericToken currentToken) {
return currentToken.getImage().isEmpty();
}
-
- @Override
- protected void analyzeToken(final GenericToken currentToken) {
- // noop
- }
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
similarity index 93%
rename from pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java
rename to pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
index 4be453b919..0f43a305fb 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
@@ -2,8 +2,9 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
-package net.sourceforge.pmd.cpd.token;
+package net.sourceforge.pmd.cpd.token.internal;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@@ -61,6 +62,17 @@ public abstract class BaseTokenFilter implements TokenFi
}
}
+ /**
+ * Extension point for subclasses to analyze all tokens (before filtering)
+ * and update internal status to decide on custom discard rules.
+ *
+ * @param currentToken The token to be analyzed
+ * @see #isLanguageSpecificDiscarding()
+ */
+ protected void analyzeToken(final GenericToken currentToken) {
+ // noop
+ }
+
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
@@ -78,13 +90,4 @@ public abstract class BaseTokenFilter implements TokenFi
*/
protected abstract boolean shouldStopProcessing(T currentToken);
- /**
- * Extension point for subclasses to analyze all tokens (before filtering)
- * and update internal status to decide on custom discard rules.
- *
- * @param currentToken The token to be analyzed
- * @see #isLanguageSpecificDiscarding()
- */
- protected abstract void analyzeToken(T currentToken);
-
}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
index 0bf95b3336..48434eca01 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
@@ -34,6 +34,14 @@ public class AntlrTokenManager implements TokenManager {
@Override
public Object getNextToken() {
+ AntlrToken nextToken = getNextTokenFromAnyChannel();
+ while (!nextToken.isDefault()) {
+ nextToken = getNextTokenFromAnyChannel();
+ }
+ return nextToken;
+ }
+
+ private AntlrToken getNextTokenFromAnyChannel() {
final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null;
final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment);
previousToken = currentToken;
diff --git a/pmd-core/src/main/resources/rulesets/releases/6120.xml b/pmd-core/src/main/resources/rulesets/releases/6120.xml
new file mode 100644
index 0000000000..873a0d1088
--- /dev/null
+++ b/pmd-core/src/main/resources/rulesets/releases/6120.xml
@@ -0,0 +1,14 @@
+
+
+
+
+This ruleset contains links to rules that are new in PMD v6.12.0
+
+
+
+
+
+
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
index 09ab82cd1a..5573c736df 100644
--- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
@@ -6,22 +6,19 @@ package net.sourceforge.pmd.cpd;
import java.io.BufferedReader;
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import java.util.Properties;
import net.sourceforge.pmd.PMD;
-import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
-import net.sourceforge.pmd.cpd.token.TokenFilter;
-import net.sourceforge.pmd.lang.ast.GenericToken;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
+import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.cpp.CppTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The C++ tokenizer.
*/
-public class CPPTokenizer implements Tokenizer {
+public class CPPTokenizer extends JavaCCTokenizer {
private boolean skipBlocks = true;
private String skipBlocksStart;
@@ -49,26 +46,6 @@ public class CPPTokenizer implements Tokenizer {
}
}
- @Override
- public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
- StringBuilder buffer = sourceCode.getCodeBuffer();
- try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) {
- final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader));
-
- GenericToken currentToken = tokenFilter.getNextToken();
- while (currentToken != null) {
- tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
- currentToken = tokenFilter.getNextToken();
- }
- tokenEntries.add(TokenEntry.getEOF());
- System.err.println("Added " + sourceCode.getFileName());
- } catch (TokenMgrError | IOException err) {
- err.printStackTrace();
- System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
- tokenEntries.add(TokenEntry.getEOF());
- }
- }
-
private String maybeSkipBlocks(String test) throws IOException {
if (!skipBlocks) {
return test;
@@ -92,4 +69,14 @@ public class CPPTokenizer implements Tokenizer {
}
return filtered.toString();
}
+
+ @Override
+ protected TokenManager getLexerForSource(SourceCode sourceCode) {
+ try {
+ StringBuilder buffer = sourceCode.getCodeBuffer();
+ return new CppTokenManager(IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString()))));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
index f5cd31bd1b..188609febd 100644
--- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
+++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
@@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -117,7 +118,7 @@ public class CPPTokenizerContinuationTest {
.getResourceAsStream("cpp/" + name), StandardCharsets.UTF_8);
}
- private Tokens parse(String code) {
+ private Tokens parse(String code) throws IOException {
CPPTokenizer tokenizer = new CPPTokenizer();
tokenizer.setProperties(new Properties());
Tokens tokens = new Tokens();
diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
index fb8fbd3360..4bffb5208d 100644
--- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
+++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
@@ -8,16 +8,23 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class CPPTokenizerTest {
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@Test
public void testUTFwithBOM() {
Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
@@ -69,21 +76,31 @@ public class CPPTokenizerTest {
@Test
public void testTokenizerWithSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
- Tokens tokens = parse(test, true);
+ Tokens tokens = parse(test, true, new Tokens());
assertEquals(19, tokens.size());
}
@Test
public void testTokenizerWithSkipBlocksPattern() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
- Tokens tokens = parse(test, true, "#if debug|#endif");
+ Tokens tokens = new Tokens();
+ try {
+ parse(test, true, "#if debug|#endif", tokens);
+ } catch (TokenMgrError ignored) {
+ // ignored
+ }
assertEquals(31, tokens.size());
}
@Test
public void testTokenizerWithoutSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
- Tokens tokens = parse(test, false);
+ Tokens tokens = new Tokens();
+ try {
+ parse(test, false, tokens);
+ } catch (TokenMgrError ignored) {
+ // ignored
+ }
assertEquals(37, tokens.size());
}
@@ -128,15 +145,33 @@ public class CPPTokenizerTest {
assertEquals(9, tokens.size());
}
+ @Test
+ public void testLexicalErrorFilename() throws Exception {
+ Properties properties = new Properties();
+ properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false));
+ String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8);
+ SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp"));
+ CPPTokenizer tokenizer = new CPPTokenizer();
+ tokenizer.setProperties(properties);
+
+ expectedException.expect(TokenMgrError.class);
+ expectedException.expectMessage("Lexical error in file issue-1559.cpp at");
+ tokenizer.tokenize(code, new Tokens());
+ }
+
private Tokens parse(String snippet) {
- return parse(snippet, false);
+ try {
+ return parse(snippet, false, new Tokens());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
- private Tokens parse(String snippet, boolean skipBlocks) {
- return parse(snippet, skipBlocks, null);
+ private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) throws IOException {
+ return parse(snippet, skipBlocks, null, tokens);
}
- private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
+ private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) throws IOException {
Properties properties = new Properties();
properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
if (skipPattern != null) {
@@ -147,7 +182,6 @@ public class CPPTokenizerTest {
tokenizer.setProperties(properties);
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
- Tokens tokens = new Tokens();
tokenizer.tokenize(code, tokens);
return tokens;
}
diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp
new file mode 100644
index 0000000000..010ec09fc6
--- /dev/null
+++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp
@@ -0,0 +1,11 @@
+namespace ABC
+{
+ namespace DEF
+ {
+
+#ifdef USE_QT
+ const char* perPixelQml = R"QML(
+)QML";
+ }
+}
+#endif // USE_QT
diff --git a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go
index 302c5a580a..a613af4286 100644
--- a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go
+++ b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go
@@ -26,4 +26,6 @@ import (
func main() {
fmt.Println(stringutil.Reverse("!selpmaxe oG ,olleH"))
-}
\ No newline at end of file
+}
+
+/* Comment */
\ No newline at end of file
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
index d758704ba3..cab722447b 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
@@ -4,21 +4,22 @@
package net.sourceforge.pmd.cpd;
+import java.io.IOException;
import java.io.StringReader;
import java.util.Deque;
import java.util.LinkedList;
import java.util.Properties;
+import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
-import net.sourceforge.pmd.lang.LanguageRegistry;
-import net.sourceforge.pmd.lang.LanguageVersionHandler;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
-import net.sourceforge.pmd.lang.java.JavaLanguageModule;
+import net.sourceforge.pmd.lang.java.JavaTokenManager;
import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;
import net.sourceforge.pmd.lang.java.ast.Token;
-public class JavaTokenizer implements Tokenizer {
+public class JavaTokenizer extends JavaCCTokenizer {
public static final String CPD_START = "\"CPD-START\"";
public static final String CPD_END = "\"CPD-END\"";
@@ -27,6 +28,8 @@ public class JavaTokenizer implements Tokenizer {
private boolean ignoreLiterals;
private boolean ignoreIdentifiers;
+ private ConstructorDetector constructorDetector;
+
public void setProperties(Properties properties) {
ignoreAnnotations = Boolean.parseBoolean(properties.getProperty(IGNORE_ANNOTATIONS, "false"));
ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
@@ -34,48 +37,42 @@ public class JavaTokenizer implements Tokenizer {
}
@Override
- public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
- final String fileName = sourceCode.getFileName();
- final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode);
- final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
-
- Token currentToken = (Token) tokenFilter.getNextToken();
- while (currentToken != null) {
- processToken(tokenEntries, fileName, currentToken, constructorDetector);
- currentToken = (Token) tokenFilter.getNextToken();
- }
- tokenEntries.add(TokenEntry.getEOF());
+ public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
+ constructorDetector = new ConstructorDetector(ignoreIdentifiers);
+ super.tokenize(sourceCode, tokenEntries);
}
- private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) {
+ @Override
+ protected TokenManager getLexerForSource(SourceCode sourceCode) {
final StringBuilder stringBuilder = sourceCode.getCodeBuffer();
- // Note that Java version is irrelevant for tokenizing
- final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME)
- .getVersion("1.4").getLanguageVersionHandler();
- final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
- .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString()));
- return new JavaTokenFilter(tokenMgr, ignoreAnnotations);
+ return new JavaTokenManager(new StringReader(stringBuilder.toString()));
}
- private void processToken(Tokens tokenEntries, String fileName, Token currentToken,
- ConstructorDetector constructorDetector) {
- String image = currentToken.image;
+ @Override
+ protected TokenFilter getTokenFilter(TokenManager tokenManager) {
+ return new JavaTokenFilter(tokenManager, ignoreAnnotations);
+ }
- constructorDetector.restoreConstructorToken(tokenEntries, currentToken);
+ @Override
+ protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String fileName) {
+ String image = currentToken.getImage();
+ Token javaToken = (Token) currentToken;
- if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL
- || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
- || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL
- || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
- image = String.valueOf(currentToken.kind);
+ constructorDetector.restoreConstructorToken(tokenEntries, javaToken);
+
+ if (ignoreLiterals && (javaToken.kind == JavaParserConstants.STRING_LITERAL
+ || javaToken.kind == JavaParserConstants.CHARACTER_LITERAL
+ || javaToken.kind == JavaParserConstants.DECIMAL_LITERAL
+ || javaToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
+ image = String.valueOf(javaToken.kind);
}
- if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
- image = String.valueOf(currentToken.kind);
+ if (ignoreIdentifiers && javaToken.kind == JavaParserConstants.IDENTIFIER) {
+ image = String.valueOf(javaToken.kind);
}
- constructorDetector.processToken(currentToken);
+ constructorDetector.processToken(javaToken);
- tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
+ return new TokenEntry(image, fileName, currentToken.getBeginLine());
}
public void setIgnoreLiterals(boolean ignore) {
diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml
index 7d346e881a..24fc4666fc 100644
--- a/pmd-java/src/main/resources/category/java/bestpractices.xml
+++ b/pmd-java/src/main/resources/category/java/bestpractices.xml
@@ -1519,6 +1519,67 @@ public class Foo {
+
+
+Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end
+of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions
+are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from
+the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is
+preserved.
+
+ 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Code sample
+ 1
+ 6
+