phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/pmd/pmd into kotlin-tokenizer-refactor

Browse Source
This commit is contained in:
lsoncini
2019-02-17 19:26:26 -03:00
parent 72f2e425b2 91e3f699f5
commit 9b12f1e3d0
28 changed files with 621 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/pages/pmd/rules/java.md
View File

@ -56,6 +56,7 @@ folder: pmd/rules
* [UseAssertSameInsteadOfAssertTrue](pmd_rules_java_bestpractices.html#useassertsameinsteadofasserttrue): This rule detects JUnit assertions in object references equality. These assertions should be made...
* [UseAssertTrueInsteadOfAssertEquals](pmd_rules_java_bestpractices.html#useasserttrueinsteadofassertequals): When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, ins...
* [UseCollectionIsEmpty](pmd_rules_java_bestpractices.html#usecollectionisempty): The isEmpty() method on java.util.Collection is provided to determine if a collection has any ele...
* [UseTryWithResources](pmd_rules_java_bestpractices.html#usetrywithresources): Java 7 introduced the try-with-resources statement. This statement ensures that each resource is ...
* [UseVarargs](pmd_rules_java_bestpractices.html#usevarargs): Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic...
## Code Style

65
docs/pages/pmd/rules/java/bestpractices.md
View File

@ -5,7 +5,7 @@ permalink: pmd_rules_java_bestpractices.html
folder: pmd/rules/java
sidebaractiveurl: /pmd_rules_java.html
editmepath: ../pmd-java/src/main/resources/category/java/bestpractices.xml
keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseVarargs
keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseTryWithResources, UseVarargs
language: Java
---
<!-- DO NOT EDIT THIS FILE. This file is generated from file ../pmd-java/src/main/resources/category/java/bestpractices.xml. -->
@ -1636,6 +1636,69 @@ public class Foo {
<rule ref="category/java/bestpractices.xml/UseCollectionIsEmpty" />
```
## UseTryWithResources
**Since:** PMD 6.12.0
**Priority:** Medium (3)
**Minimum Language Version:** Java 1.7
Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end
of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions
are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from
the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is
preserved.
**This rule is defined by the following XPath expression:**
``` xpath
//TryStatement[FinallyStatement//Name[
tokenize(@Image, '\.')[last()] = $closeMethods
][
pmd-java:typeIs('java.lang.AutoCloseable')
or
../../PrimarySuffix/Arguments[@ArgumentCount = 1]//PrimaryPrefix[pmd-java:typeIs('java.lang.AutoCloseable')]
]]
```
**Example(s):**
``` java
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) in.close();
} catch (IOException ignored) {
// ignored
}
}
// better use try-with-resources
try (InputStream in2 = openInputStream()) {
int i = in2.read();
}
}
}
```
**This rule has the following properties:**
|Name|Default Value|Description|Multivalued|
|----|-------------|-----------|-----------|
|closeMethods|close , closeQuietly|Method names in finally block, which trigger this rule|yes. Delimiter is ','.|
**Use this rule by referencing it:**
``` xml
<rule ref="category/java/bestpractices.xml/UseTryWithResources" />
```
## UseVarargs
**Since:** PMD 5.0

4
docs/pages/pmd/userdocs/cpd.md
View File

@ -359,8 +359,8 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package:
## Suppression
Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Javascript**, **Matlab**,
**Objective-C**, **PL/SQL** and **Python** by including the keywords `CPD-OFF` and `CPD-ON`.
Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**,
**Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`.
```java
public Object someParameterizedFactoryMethod(int x) throws Exception {

28
docs/pages/release_notes.md
View File

@ -14,6 +14,30 @@ This is a {{ site.pmd.release_type }} release.
### New and noteworthy
### CPD Suppression for Antlr-based languages
[ITBA](https://www.itba.edu.ar/) students [Matías Fraga](https://github.com/matifraga),
[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini)
keep working on bringing full Antlr support to PMD. For this release, they have implemented
token filtering in an equivalent way as we did for JavaCC languages, adding support for CPD
suppressions through `CPD-OFF` and `CPD-ON` comments for all Antlr-based languages.
This means, you can now ignore arbitrary blocks of code on:
* Go
* Kotlin
* Swift
Simply start the suppression with any comment (single or multiline) containing `CPD-OFF`,
and resume again with a comment containing `CPD-ON`.
More information is available in [the user documentation](pmd_userdocs_cpd.html#suppression).
#### New Rules
* The new Java rule {% rule "java/bestpractices/UseTryWithResources" %) (`java-bestpractices`) searches
for try-blocks, that could be changed to a try-with-resources statement. This statement ensures that
each resource is closed at the end of the statement and is available since Java 7.
#### Modified Rules
* The Apex rule {% rule "apex/codestyle/MethodNamingConventions" %} (apex-codestyle) has a new
@ -22,8 +46,11 @@ This is a {{ site.pmd.release_type }} release.
### Fixed Issues
* all
* [#1559](https://github.com/pmd/pmd/issues/1559): \[core] CPD: Lexical error in file (no file name provided)
* java-bestpractices
* [#808](https://github.com/pmd/pmd/issues/808): \[java] AccessorMethodGeneration false positives with compile time constants
* [#1405](https://github.com/pmd/pmd/issues/1405): \[java] New Rule: UseTryWithResources - Replace close and IOUtils.closeQuietly with try-with-resources
* [#1555](https://github.com/pmd/pmd/issues/1555): \[java] UnusedImports false positive for method parameter type in @see Javadoc
* java-codestyle
* [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods
@ -52,6 +79,7 @@ This is a {{ site.pmd.release_type }} release.
* [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly)
* [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17)
* [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17)
* [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca)
{% endtocmaker %}

7
pmd-core/src/main/java/net/sourceforge/pmd/cpd/AntlrTokenizer.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
@ -14,7 +15,11 @@ import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Generic implementation of a {@link Tokenizer} useful to any Antlr grammar.
*
* @deprecated This is an internal API.
*/
@Deprecated
@InternalApi
public abstract class AntlrTokenizer implements Tokenizer {
protected abstract AntlrTokenManager getLexerForSource(SourceCode sourceCode);
@ -23,6 +28,8 @@ public abstract class AntlrTokenizer implements Tokenizer {
public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) {
final AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
tokenManager.setFileName(sourceCode.getFileName());
final AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager);
try {

46
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java Normal file
View File

@ -0,0 +1,46 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.internal;
import java.io.IOException;
import net.sourceforge.pmd.cpd.SourceCode;
import net.sourceforge.pmd.cpd.TokenEntry;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
public abstract class JavaCCTokenizer implements Tokenizer {
protected abstract TokenManager getLexerForSource(SourceCode sourceCode);
protected TokenFilter getTokenFilter(TokenManager tokenManager) {
return new JavaCCTokenFilter(tokenManager);
}
protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) {
return new TokenEntry(currentToken.getImage(), filename, currentToken.getBeginLine());
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
TokenManager tokenManager = getLexerForSource(sourceCode);
tokenManager.setFileName(sourceCode.getFileName());
try {
final TokenFilter tokenFilter = getTokenFilter(tokenManager);
GenericToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName()));
currentToken = tokenFilter.getNextToken();
}
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
}
}

4
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrToken.java
View File

@ -71,4 +71,8 @@ public class AntlrToken implements GenericToken {
public boolean isHidden() {
return token.getChannel() == Lexer.HIDDEN;
}
public boolean isDefault() {
return token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL;
}
}

17
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/AntlrTokenFilter.java
View File

@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd.token;
import static org.antlr.v4.runtime.Token.EOF;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
/**
@ -14,8 +15,6 @@ import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
*/
public class AntlrTokenFilter extends BaseTokenFilter<AntlrToken> {
private boolean discardingHiddenTokens = false;
/**
* Creates a new AntlrTokenFilter
* @param tokenManager The token manager from which to retrieve tokens to be filtered
@ -28,18 +27,4 @@ public class AntlrTokenFilter extends BaseTokenFilter<AntlrToken> {
protected boolean shouldStopProcessing(final AntlrToken currentToken) {
return currentToken.getType() == EOF;
}
@Override
protected void analyzeToken(final AntlrToken currentToken) {
analyzeHiddenTokens(currentToken);
}
@Override
protected boolean isLanguageSpecificDiscarding() {
return super.isLanguageSpecificDiscarding() || discardingHiddenTokens;
}
private void analyzeHiddenTokens(final AntlrToken token) {
discardingHiddenTokens = token.isHidden();
}
}

6
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
View File

@ -4,6 +4,7 @@
package net.sourceforge.pmd.cpd.token;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@ -25,9 +26,4 @@ public class JavaCCTokenFilter extends BaseTokenFilter<GenericToken> {
protected boolean shouldStopProcessing(final GenericToken currentToken) {
return currentToken.getImage().isEmpty();
}
@Override
protected void analyzeToken(final GenericToken currentToken) {
// noop
}
}

23
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/BaseTokenFilter.java → pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
View File

@ -2,8 +2,9 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token;
package net.sourceforge.pmd.cpd.token.internal;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@ -61,6 +62,17 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
}
}
/**
* Extension point for subclasses to analyze all tokens (before filtering)
* and update internal status to decide on custom discard rules.
*
* @param currentToken The token to be analyzed
* @see #isLanguageSpecificDiscarding()
*/
protected void analyzeToken(final GenericToken currentToken) {
// noop
}
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
@ -78,13 +90,4 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
*/
protected abstract boolean shouldStopProcessing(T currentToken);
/**
* Extension point for subclasses to analyze all tokens (before filtering)
* and update internal status to decide on custom discard rules.
*
* @param currentToken The token to be analyzed
* @see #isLanguageSpecificDiscarding()
*/
protected abstract void analyzeToken(T currentToken);
}

8
pmd-core/src/main/java/net/sourceforge/pmd/lang/antlr/AntlrTokenManager.java
View File

@ -34,6 +34,14 @@ public class AntlrTokenManager implements TokenManager {
@Override
public Object getNextToken() {
AntlrToken nextToken = getNextTokenFromAnyChannel();
while (!nextToken.isDefault()) {
nextToken = getNextTokenFromAnyChannel();
}
return nextToken;
}
private AntlrToken getNextTokenFromAnyChannel() {
final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null;
final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment);
previousToken = currentToken;

14
pmd-core/src/main/resources/rulesets/releases/6120.xml Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0"?>
<ruleset name="6120"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
<description>
This ruleset contains links to rules that are new in PMD v6.12.0
</description>
<rule ref="category/java/bestpractices.xml/UseTryWithResources"/>
</ruleset>

39
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -6,22 +6,19 @@ package net.sourceforge.pmd.cpd;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Properties;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.cpp.CppTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The C++ tokenizer.
*/
public class CPPTokenizer implements Tokenizer {
public class CPPTokenizer extends JavaCCTokenizer {
private boolean skipBlocks = true;
private String skipBlocksStart;
@ -49,26 +46,6 @@ public class CPPTokenizer implements Tokenizer {
}
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader));
GenericToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
currentToken = tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError | IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
}
}
private String maybeSkipBlocks(String test) throws IOException {
if (!skipBlocks) {
return test;
@ -92,4 +69,14 @@ public class CPPTokenizer implements Tokenizer {
}
return filtered.toString();
}
@Override
protected TokenManager getLexerForSource(SourceCode sourceCode) {
try {
StringBuilder buffer = sourceCode.getCodeBuffer();
return new CppTokenManager(IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString()))));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

3
pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@ -117,7 +118,7 @@ public class CPPTokenizerContinuationTest {
.getResourceAsStream("cpp/" + name), StandardCharsets.UTF_8);
}
private Tokens parse(String code) {
private Tokens parse(String code) throws IOException {
CPPTokenizer tokenizer = new CPPTokenizer();
tokenizer.setProperties(new Properties());
Tokens tokens = new Tokens();

50
pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
View File

@ -8,16 +8,23 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class CPPTokenizerTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testUTFwithBOM() {
Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
@ -69,21 +76,31 @@ public class CPPTokenizerTest {
@Test
public void testTokenizerWithSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, true);
Tokens tokens = parse(test, true, new Tokens());
assertEquals(19, tokens.size());
}
@Test
public void testTokenizerWithSkipBlocksPattern() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, true, "#if debug|#endif");
Tokens tokens = new Tokens();
try {
parse(test, true, "#if debug|#endif", tokens);
} catch (TokenMgrError ignored) {
// ignored
}
assertEquals(31, tokens.size());
}
@Test
public void testTokenizerWithoutSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, false);
Tokens tokens = new Tokens();
try {
parse(test, false, tokens);
} catch (TokenMgrError ignored) {
// ignored
}
assertEquals(37, tokens.size());
}
@ -128,15 +145,33 @@ public class CPPTokenizerTest {
assertEquals(9, tokens.size());
}
@Test
public void testLexicalErrorFilename() throws Exception {
Properties properties = new Properties();
properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false));
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8);
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp"));
CPPTokenizer tokenizer = new CPPTokenizer();
tokenizer.setProperties(properties);
expectedException.expect(TokenMgrError.class);
expectedException.expectMessage("Lexical error in file issue-1559.cpp at");
tokenizer.tokenize(code, new Tokens());
}
private Tokens parse(String snippet) {
return parse(snippet, false);
try {
return parse(snippet, false, new Tokens());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Tokens parse(String snippet, boolean skipBlocks) {
return parse(snippet, skipBlocks, null);
private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) throws IOException {
return parse(snippet, skipBlocks, null, tokens);
}
private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) throws IOException {
Properties properties = new Properties();
properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
if (skipPattern != null) {
@ -147,7 +182,6 @@ public class CPPTokenizerTest {
tokenizer.setProperties(properties);
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
Tokens tokens = new Tokens();
tokenizer.tokenize(code, tokens);
return tokens;
}

11
pmd-cpp/src/test/resources/net/sourceforge/pmd/cpd/cpp/issue-1559.cpp Normal file
View File

@ -0,0 +1,11 @@
namespace ABC
{
namespace DEF
{
#ifdef USE_QT
const char* perPixelQml = R"QML(
)QML";
}
}
#endif // USE_QT

4
pmd-go/src/test/resources/net/sourceforge/pmd/cpd/hello.go
View File

@ -26,4 +26,6 @@ import (
func main() {
fmt.Println(stringutil.Reverse("!selpmaxe oG ,olleH"))
}
}
/* Comment */

67
pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
View File

@ -4,21 +4,22 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.StringReader;
import java.util.Deque;
import java.util.LinkedList;
import java.util.Properties;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
import net.sourceforge.pmd.lang.java.JavaTokenManager;
import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;
import net.sourceforge.pmd.lang.java.ast.Token;
public class JavaTokenizer implements Tokenizer {
public class JavaTokenizer extends JavaCCTokenizer {
public static final String CPD_START = "\"CPD-START\"";
public static final String CPD_END = "\"CPD-END\"";
@ -27,6 +28,8 @@ public class JavaTokenizer implements Tokenizer {
private boolean ignoreLiterals;
private boolean ignoreIdentifiers;
private ConstructorDetector constructorDetector;
public void setProperties(Properties properties) {
ignoreAnnotations = Boolean.parseBoolean(properties.getProperty(IGNORE_ANNOTATIONS, "false"));
ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
@ -34,48 +37,42 @@ public class JavaTokenizer implements Tokenizer {
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
final String fileName = sourceCode.getFileName();
final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode);
final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
processToken(tokenEntries, fileName, currentToken, constructorDetector);
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
constructorDetector = new ConstructorDetector(ignoreIdentifiers);
super.tokenize(sourceCode, tokenEntries);
}
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) {
@Override
protected TokenManager getLexerForSource(SourceCode sourceCode) {
final StringBuilder stringBuilder = sourceCode.getCodeBuffer();
// Note that Java version is irrelevant for tokenizing
final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME)
.getVersion("1.4").getLanguageVersionHandler();
final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString()));
return new JavaTokenFilter(tokenMgr, ignoreAnnotations);
return new JavaTokenManager(new StringReader(stringBuilder.toString()));
}
private void processToken(Tokens tokenEntries, String fileName, Token currentToken,
ConstructorDetector constructorDetector) {
String image = currentToken.image;
@Override
protected TokenFilter getTokenFilter(TokenManager tokenManager) {
return new JavaTokenFilter(tokenManager, ignoreAnnotations);
}
constructorDetector.restoreConstructorToken(tokenEntries, currentToken);
@Override
protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String fileName) {
String image = currentToken.getImage();
Token javaToken = (Token) currentToken;
if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL
|| currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
|| currentToken.kind == JavaParserConstants.DECIMAL_LITERAL
|| currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
image = String.valueOf(currentToken.kind);
constructorDetector.restoreConstructorToken(tokenEntries, javaToken);
if (ignoreLiterals && (javaToken.kind == JavaParserConstants.STRING_LITERAL
|| javaToken.kind == JavaParserConstants.CHARACTER_LITERAL
|| javaToken.kind == JavaParserConstants.DECIMAL_LITERAL
|| javaToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
image = String.valueOf(javaToken.kind);
}
if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
image = String.valueOf(currentToken.kind);
if (ignoreIdentifiers && javaToken.kind == JavaParserConstants.IDENTIFIER) {
image = String.valueOf(javaToken.kind);
}
constructorDetector.processToken(currentToken);
constructorDetector.processToken(javaToken);
tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
return new TokenEntry(image, fileName, currentToken.getBeginLine());
}
public void setIgnoreLiterals(boolean ignore) {

61
pmd-java/src/main/resources/category/java/bestpractices.xml
View File

@ -1519,6 +1519,67 @@ public class Foo {
</example>
</rule>
<rule name="UseTryWithResources"
language="java"
minimumLanguageVersion="1.7"
since="6.12.0"
typeResolution="true"
message="Consider using a try-with-resources statement instead of explicitly closing the resource"
class="net.sourceforge.pmd.lang.rule.XPathRule"
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_bestpractices.html#usetrywithresources">
<description>
Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end
of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions
are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from
the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is
preserved.
</description>
<priority>3</priority>
<properties>
<property name="closeMethods" type="List[String]" delimiter="," description="Method names in finally block, which trigger this rule" value="close,closeQuietly"/>
<property name="version" value="2.0" />
<property name="xpath">
<value>
<![CDATA[
//TryStatement[FinallyStatement//Name[
tokenize(@Image, '\.')[last()] = $closeMethods
][
pmd-java:typeIs('java.lang.AutoCloseable')
or
../../PrimarySuffix/Arguments[@ArgumentCount = 1]//PrimaryPrefix[pmd-java:typeIs('java.lang.AutoCloseable')]
]]
]]>
</value>
</property>
</properties>
<example>
<![CDATA[
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) in.close();
} catch (IOException ignored) {
// ignored
}
}
// better use try-with-resources
try (InputStream in2 = openInputStream()) {
int i = in2.read();
}
}
}
]]>
</example>
</rule>
<rule name="UseVarargs"
language="java"
minimumLanguageVersion="1.5"

2
pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokensTokenizerTest.java
View File

@ -88,7 +88,7 @@ public class JavaTokensTokenizerTest {
}
@Test
public void testIgnoreComments() {
public void testIgnoreComments() throws IOException {
JavaTokenizer t = new JavaTokenizer();
t.setIgnoreAnnotations(false);
SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL

5
pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@ -32,7 +33,7 @@ public class MatchAlgorithmTest {
}
@Test
public void testSimple() {
public void testSimple() throws IOException {
JavaTokenizer tokenizer = new JavaTokenizer();
SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(getSampleCode(), "Foo.java"));
Tokens tokens = new Tokens();
@ -63,7 +64,7 @@ public class MatchAlgorithmTest {
}
@Test
public void testIgnore() {
public void testIgnore() throws IOException {
JavaTokenizer tokenizer = new JavaTokenizer();
tokenizer.setIgnoreLiterals(true);
tokenizer.setIgnoreIdentifiers(true);

11
pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/bestpractices/UseTryWithResourcesTest.java Normal file
View File

@ -0,0 +1,11 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.java.rule.bestpractices;
import net.sourceforge.pmd.testframework.PmdRuleTst;
public class UseTryWithResourcesTest extends PmdRuleTst {
// no additional unit tests
}

192
pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UseTryWithResources.xml Normal file
View File

@ -0,0 +1,192 @@
<?xml version="1.0" encoding="UTF-8"?>
<test-data
xmlns="http://pmd.sourceforge.net/rule-tests"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/rule-tests http://pmd.sourceforge.net/rule-tests_1_0_0.xsd">
<test-code>
<description>Code sample</description>
<expected-problems>1</expected-problems>
<expected-linenumbers>6</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) in.close();
} catch (IOException ignored) {
// ignored
}
}
// better use try-with-resources
try (InputStream in2 = openInputStream()) {
int i = in2.read();
}
}
}
]]></code>
</test-code>
<test-code>
<description>With IOUtils.closeQuietly 1</description>
<expected-problems>1</expected-problems>
<expected-linenumbers>6</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
IOUtils.closeQuietly(in);
}
}
}
]]></code>
</test-code>
<test-code>
<description>With IOUtils.closeQuietly 2</description>
<expected-problems>1</expected-problems>
<expected-linenumbers>6</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
closeQuietly(in);
}
}
}
]]></code>
</test-code>
<test-code>
<description>Multiple Resources</description>
<expected-problems>1</expected-problems>
<expected-linenumbers>7</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
InputStream in1 = null;
InputStream in2 = null;
try {
in1 = openInputStream();
in2 = openInputStream();
int x = in1.read();
int y = in2.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in1 != null) in1.close();
} catch (IOException ignored) {
// ignored
}
IOUtils.closeQuietly(in2);
}
// better use try-with-resources
try (InputStream in3 = openInputStream(); InputStream in4 = openInputStream()) {
int x = in3.read();
int y = in4.read();
}
}
}
]]></code>
</test-code>
<test-code>
<description>Custom close methods</description>
<rule-property name="closeMethods">myClose2,myClose</rule-property>
<expected-problems>1</expected-problems>
<expected-linenumbers>6</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
myClose(in);
}
// this block doesn't trigger the rule because of the custom close methods property
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
IOUtils.closeQuietly(in);
}
}
}
]]></code>
</test-code>
<test-code>
<description>False negative with two resources</description>
<expected-problems>1</expected-problems>
<expected-linenumbers>6</expected-linenumbers>
<code><![CDATA[
import java.io.*;
public class TryWithResources {
public void run() {
Reader r1 = new FileReader(file);
try (Reader r2 = new FileReader(otherFile)) {
} finally {
r1.close();
}
}
}
]]></code>
</test-code>
<test-code>
<description>False positive with no autocloseable</description>
<expected-problems>0</expected-problems>
<code><![CDATA[
public class TryWithResources {
public void run() {
var noAutoclosable = new Object() {
public void close() {}
};
try {
System.out.println(noAutocloseable);
} finally {
noAutoclosable.close();
}
}
}
]]></code>
</test-code>
</test-data>

55
pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
View File

@ -4,56 +4,39 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ecmascript5.Ecmascript5TokenManager;
import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants;
import net.sourceforge.pmd.lang.ecmascript5.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Ecmascript Tokenizer
*/
public class EcmascriptTokenizer implements Tokenizer {
public class EcmascriptTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = new StringReader(buffer.toString())) {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(
new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} catch (IOException e) {
e.printStackTrace();
}
return new Ecmascript5TokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
private String getTokenImage(Token token) {
@Override
protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) {
return new TokenEntry(getTokenImage(currentToken), filename, currentToken.getBeginLine());
}
private String getTokenImage(GenericToken token) {
Token jsToken = (Token) token;
// Remove line continuation characters from string literals
if (token.kind == Ecmascript5ParserConstants.STRING_LITERAL
|| token.kind == Ecmascript5ParserConstants.UNTERMINATED_STRING_LITERAL) {
return token.image.replaceAll("(?<!\\\\)\\\\(\\r\\n|\\r|\\n)", "");
if (jsToken.kind == Ecmascript5ParserConstants.STRING_LITERAL
|| jsToken.kind == Ecmascript5ParserConstants.UNTERMINATED_STRING_LITERAL) {
return token.getImage().replaceAll("(?<!\\\\)\\\\(\\r\\n|\\r|\\n)", "");
}
return token.image;
return token.getImage();
}
}

27
pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
View File

@ -4,38 +4,21 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.matlab.MatlabTokenManager;
import net.sourceforge.pmd.lang.matlab.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Matlab Tokenizer.
*/
public class MatlabTokenizer implements Tokenizer {
public class MatlabTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new MatlabTokenManager(reader));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError | IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
}
return new MatlabTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}

30
pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
View File

@ -4,39 +4,21 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.objectivec.ObjectiveCTokenManager;
import net.sourceforge.pmd.lang.objectivec.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Objective-C Tokenizer
*/
public class ObjectiveCTokenizer implements Tokenizer {
public class ObjectiveCTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = new StringReader(buffer.toString())) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new ObjectiveCTokenManager(reader));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} catch (IOException e) {
e.printStackTrace();
}
return new ObjectiveCTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}

79
pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
View File

@ -6,18 +6,16 @@ package net.sourceforge.pmd.cpd;
import java.io.StringReader;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager;
import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants;
import net.sourceforge.pmd.lang.plsql.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
public class PLSQLTokenizer implements Tokenizer {
private static final Logger LOGGER = Logger.getLogger(PLSQLTokenizer.class.getName());
public class PLSQLTokenizer extends JavaCCTokenizer {
// This is actually useless, the comments are special tokens, never taken into account by CPD
@Deprecated
public static final String IGNORE_COMMENTS = "ignore_comments";
@ -51,58 +49,31 @@ public class PLSQLTokenizer implements Tokenizer {
this.ignoreIdentifiers = ignore;
}
/**
* Read Reader from SourceCode and output an ordered tree of PLSQL tokens.
*
* @param sourceCode
* PLSQL source in file, string or database (any suitable object
* that can return a Reader).
* @param tokenEntries
* Derived based on PLSQL Abstract Syntax Tree (derived from
* PLDOc parser.)
*/
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
long encounteredTokens = 0;
long addedTokens = 0;
protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String fileName) {
String image = currentToken.getImage();
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("PLSQLTokenizer: ignoreIdentifiers==" + ignoreIdentifiers);
LOGGER.fine("PLSQLTokenizer: ignoreLiterals==" + ignoreLiterals);
Token plsqlToken = (Token) currentToken;
if (ignoreIdentifiers && plsqlToken.kind == PLSQLParserConstants.IDENTIFIER) {
image = String.valueOf(plsqlToken.kind);
}
String fileName = sourceCode.getFileName();
StringBuilder sb = sourceCode.getCodeBuffer();
TokenFilter tokenFilter = new JavaCCTokenFilter(new PLSQLTokenManager(new StringReader(sb.toString())));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
String image = currentToken.image;
encounteredTokens++;
if (ignoreIdentifiers && currentToken.kind == PLSQLParserConstants.IDENTIFIER) {
image = String.valueOf(currentToken.kind);
}
if (ignoreLiterals && (currentToken.kind == PLSQLParserConstants.UNSIGNED_NUMERIC_LITERAL
|| currentToken.kind == PLSQLParserConstants.FLOAT_LITERAL
|| currentToken.kind == PLSQLParserConstants.INTEGER_LITERAL
|| currentToken.kind == PLSQLParserConstants.CHARACTER_LITERAL
|| currentToken.kind == PLSQLParserConstants.STRING_LITERAL
|| currentToken.kind == PLSQLParserConstants.QUOTED_LITERAL)) {
image = String.valueOf(currentToken.kind);
}
tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
addedTokens++;
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine(sourceCode.getFileName() + ": encountered " + encounteredTokens + " tokens;" + " added "
+ addedTokens + " tokens");
if (ignoreLiterals && (plsqlToken.kind == PLSQLParserConstants.UNSIGNED_NUMERIC_LITERAL
|| plsqlToken.kind == PLSQLParserConstants.FLOAT_LITERAL
|| plsqlToken.kind == PLSQLParserConstants.INTEGER_LITERAL
|| plsqlToken.kind == PLSQLParserConstants.CHARACTER_LITERAL
|| plsqlToken.kind == PLSQLParserConstants.STRING_LITERAL
|| plsqlToken.kind == PLSQLParserConstants.QUOTED_LITERAL)) {
image = String.valueOf(plsqlToken.kind);
}
return new TokenEntry(image, fileName, currentToken.getBeginLine());
}
@Override
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder stringBuilder = sourceCode.getCodeBuffer();
return new PLSQLTokenManager(IOUtil.skipBOM(new StringReader(stringBuilder.toString())));
}
}

35
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
View File

@ -4,44 +4,21 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.python.PythonTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Python tokenizer.
*/
public class PythonTokenizer implements Tokenizer {
public class PythonTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader));
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode);
} catch (TokenMgrError | IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
}
return new PythonTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}