Merge branch 'master' into designer-event-refactor

This commit is contained in:
Clément Fournier 2019-02-18 10:12:15 +01:00
commit 74339cae54
106 changed files with 2891 additions and 782 deletions

View File

@ -35,10 +35,11 @@ function upload_baseline() {
cd target/reports
BRANCH_FILENAME="${TRAVIS_BRANCH/\//_}"
zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/
rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/
../../pmd/.travis/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/"
if [ $? -ne 0 ]; then
log_error "Error while uploading ${BRANCH_FILENAME}-baseline.zip to sourceforge!"
log_error "Please upload manually: https://sourceforge.net/projects/pmd/files/pmd-regression-tester/"
exit 1
else
log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to sourceforge"
fi
@ -89,10 +90,11 @@ elif travis_isPush; then
echo -e "\n\n"
log_info "Uploading pmd distribution to sourceforge..."
rsync -avh pmd-dist/target/pmd-*-${VERSION}.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${VERSION}/
.travis/travis_wait "rsync -avh pmd-dist/target/pmd-*-${VERSION}.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${VERSION}/"
if [ $? -ne 0 ]; then
log_error "Error while uploading pmd-*-${VERSION}.zip to sourceforge!"
log_error "Please upload manually: https://sourceforge.net/projects/pmd/files/pmd/"
exit 1
else
log_success "Successfully uploaded pmd-*-${VERSION}.zip to sourceforge"
fi
@ -127,8 +129,6 @@ elif travis_isPush; then
set +e
upload_baseline
true
)
else

View File

@ -35,7 +35,7 @@ ls -lh pmd-doc-${VERSION}.zip
if [[ "${TRAVIS_TAG}" != "" || "${VERSION}" == *-SNAPSHOT ]]; then
echo -e "\n\n"
log_info "Uploading pmd doc distribution to sourceforge..."
rsync -avh pmd-doc-${VERSION}.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${VERSION}/
../.travis/travis_wait "rsync -avh pmd-doc-${VERSION}.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${VERSION}/"
if [ $? -ne 0 ]; then
log_error "Couldn't upload pmd-doc-${VERSION}.zip!"
log_error "Please upload manually: https://sourceforge.net/projects/pmd/files/pmd/"
@ -48,7 +48,7 @@ ls -lh pmd-doc-${VERSION}.zip
if [[ "${VERSION}" == *-SNAPSHOT && "${TRAVIS_BRANCH}" == "master" ]] && has_docs_change; then
echo -e "\n\n"
log_info "Uploading snapshot site to pmd.sourceforge.net/snapshot..."
travis_wait rsync -ah --stats --delete pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/
../.travis/travis_wait "rsync -ah --stats --delete pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/"
if [ $? -ne 0 ]; then
log_error "Couldn't upload the snapshot documentation. It won't be current on http://pmd.sourceforge.net/snapshot/"
else

View File

@ -314,6 +314,8 @@ main() {
else
exit_slowpoke=${exit_force}
fi
# Output last couple of lines from $file_log
tail -5 "${file_log}"
show_warning "Your given command has terminated with exit code $exit_slowpoke. So do I."
exit ${exit_slowpoke}

View File

@ -187,6 +187,7 @@ public class Foo {
|cc\_categories|Style|Code Climate Categories|yes. Delimiter is '\|'.|
|cc\_remediation\_points\_multiplier|1|Code Climate Remediation Points multiplier|no|
|cc\_block\_highlighting|false|Code Climate Block Highlighting|no|
|skipTestMethodUnderscores|false|Skip underscores in test methods|no|
**Use this rule by referencing it:**
``` xml

View File

@ -56,6 +56,7 @@ folder: pmd/rules
* [UseAssertSameInsteadOfAssertTrue](pmd_rules_java_bestpractices.html#useassertsameinsteadofasserttrue): This rule detects JUnit assertions in object references equality. These assertions should be made...
* [UseAssertTrueInsteadOfAssertEquals](pmd_rules_java_bestpractices.html#useasserttrueinsteadofassertequals): When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, ins...
* [UseCollectionIsEmpty](pmd_rules_java_bestpractices.html#usecollectionisempty): The isEmpty() method on java.util.Collection is provided to determine if a collection has any ele...
* [UseTryWithResources](pmd_rules_java_bestpractices.html#usetrywithresources): Java 7 introduced the try-with-resources statement. This statement ensures that each resource is ...
* [UseVarargs](pmd_rules_java_bestpractices.html#usevarargs): Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic...
## Code Style

View File

@ -5,7 +5,7 @@ permalink: pmd_rules_java_bestpractices.html
folder: pmd/rules/java
sidebaractiveurl: /pmd_rules_java.html
editmepath: ../pmd-java/src/main/resources/category/java/bestpractices.xml
keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseVarargs
keywords: Best Practices, AbstractClassWithoutAbstractMethod, AccessorClassGeneration, AccessorMethodGeneration, ArrayIsStoredDirectly, AvoidPrintStackTrace, AvoidReassigningLoopVariables, AvoidReassigningParameters, AvoidStringBufferField, AvoidUsingHardCodedIP, CheckResultSet, ConstantsInInterface, DefaultLabelNotLastInSwitchStmt, ForLoopCanBeForeach, ForLoopVariableCount, GuardLogStatement, JUnit4SuitesShouldUseSuiteAnnotation, JUnit4TestShouldUseAfterAnnotation, JUnit4TestShouldUseBeforeAnnotation, JUnit4TestShouldUseTestAnnotation, JUnitAssertionsShouldIncludeMessage, JUnitTestContainsTooManyAsserts, JUnitTestsShouldIncludeAssert, JUnitUseExpected, LooseCoupling, MethodReturnsInternalArray, MissingOverride, OneDeclarationPerLine, PositionLiteralsFirstInCaseInsensitiveComparisons, PositionLiteralsFirstInComparisons, PreserveStackTrace, ReplaceEnumerationWithIterator, ReplaceHashtableWithMap, ReplaceVectorWithList, SwitchStmtsShouldHaveDefault, SystemPrintln, UnusedFormalParameter, UnusedImports, UnusedLocalVariable, UnusedPrivateField, UnusedPrivateMethod, UseAssertEqualsInsteadOfAssertTrue, UseAssertNullInsteadOfAssertTrue, UseAssertSameInsteadOfAssertTrue, UseAssertTrueInsteadOfAssertEquals, UseCollectionIsEmpty, UseTryWithResources, UseVarargs
language: Java
---
<!-- DO NOT EDIT THIS FILE. This file is generated from file ../pmd-java/src/main/resources/category/java/bestpractices.xml. -->
@ -1636,6 +1636,69 @@ public class Foo {
<rule ref="category/java/bestpractices.xml/UseCollectionIsEmpty" />
```
## UseTryWithResources
**Since:** PMD 6.12.0
**Priority:** Medium (3)
**Minimum Language Version:** Java 1.7
Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end
of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions
are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from
the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is
preserved.
**This rule is defined by the following XPath expression:**
``` xpath
//TryStatement[FinallyStatement//Name[
tokenize(@Image, '\.')[last()] = $closeMethods
][
pmd-java:typeIs('java.lang.AutoCloseable')
or
../../PrimarySuffix/Arguments[@ArgumentCount = 1]//PrimaryPrefix[pmd-java:typeIs('java.lang.AutoCloseable')]
]]
```
**Example(s):**
``` java
public class TryWithResources {
public void run() {
InputStream in = null;
try {
in = openInputStream();
int i = in.read();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) in.close();
} catch (IOException ignored) {
// ignored
}
}
// better use try-with-resources
try (InputStream in2 = openInputStream()) {
int i = in2.read();
}
}
}
```
**This rule has the following properties:**
|Name|Default Value|Description|Multivalued|
|----|-------------|-----------|-----------|
|closeMethods|close , closeQuietly|Method names in finally block, which trigger this rule|yes. Delimiter is ','.|
**Use this rule by referencing it:**
``` xml
<rule ref="category/java/bestpractices.xml/UseTryWithResources" />
```
## UseVarargs
**Since:** PMD 5.0

View File

@ -77,7 +77,7 @@ public class Foo {
|Name|Default Value|Description|Multivalued|
|----|-------------|-----------|-----------|
|ignoredAnnotations|lombok.Data \| lombok.Value \| lombok.Builder \| lombok.NoArgsConstructor \| lombok.RequiredArgsConstructor \| lombok.AllArgsConstructorAtLeastOneConstructor|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.|
|ignoredAnnotations|lombok.Data \| lombok.Value \| lombok.Builder \| lombok.NoArgsConstructor \| lombok.RequiredArgsConstructor \| lombok.AllArgsConstructor|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.|
**Use this rule by referencing it:**
``` xml
@ -1160,6 +1160,7 @@ public class LinguisticNaming {
|Name|Default Value|Description|Multivalued|
|----|-------------|-----------|-----------|
|ignoredAnnotations|java.lang.Override|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.|
|checkBooleanMethod|true|Check method names and types for inconsistent naming.|no|
|checkGetters|true|Check return type of getters.|no|
|checkSetters|true|Check return type of setters.|no|
@ -2109,7 +2110,7 @@ which makes the code also more readable.
**This rule is defined by the following XPath expression:**
``` xpath
//VariableInitializer
//VariableInitializer[preceding-sibling::VariableDeclaratorId[1]/@TypeInferred="false"]
//PrimaryExpression[not(PrimarySuffix)]
[not(ancestor::ArgumentList)]
/PrimaryPrefix/AllocationExpression[ClassOrInterfaceType[@AnonymousClass='false']/TypeArguments//ReferenceType[not(.//TypeArguments)]]

View File

@ -738,6 +738,7 @@ private int getMoreFoo(){
|Name|Default Value|Description|Multivalued|
|----|-------------|-----------|-----------|
|ignoredAnnotations|lombok.Data \| lombok.Getter \| lombok.Value|Fully qualified names of the annotation types that should be ignored by this rule|yes. Delimiter is '\|'.|
|prefix||A variable prefix to skip, i.e., m\_|no|
**Use this rule by referencing it:**

View File

@ -359,8 +359,8 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package:
## Suppression
Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Javascript**, **Matlab**,
**Objective-C**, **PL/SQL** and **Python** by including the keywords `CPD-OFF` and `CPD-ON`.
Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**,
**Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`.
```java
public Object someParameterizedFactoryMethod(int x) throws Exception {

View File

@ -14,11 +14,86 @@ This is a {{ site.pmd.release_type }} release.
### New and noteworthy
### CPD Suppression for Antlr-based languages
[ITBA](https://www.itba.edu.ar/) students [Matías Fraga](https://github.com/matifraga),
[Tomi De Lucca](https://github.com/tomidelucca) and [Lucas Soncini](https://github.com/lsoncini)
keep working on bringing full Antlr support to PMD. For this release, they have implemented
token filtering in an equivalent way as we did for JavaCC languages, adding support for CPD
suppressions through `CPD-OFF` and `CPD-ON` comments for all Antlr-based languages.
This means, you can now ignore arbitrary blocks of code on:
* Go
* Kotlin
* Swift
Simply start the suppression with any comment (single or multiline) containing `CPD-OFF`,
and resume again with a comment containing `CPD-ON`.
More information is available in [the user documentation](pmd_userdocs_cpd.html#suppression).
#### PL/SQL Grammar improvements
* In this release, many more parser bugs in our PL/SQL support have been fixed. This adds more complete
support for UPDATE statements and subqueries and hierarchical queries in SELECT statements.
* Support for analytic functions such as LISTAGG has been added.
* Conditions in WHERE clauses support now REGEX_LIKE and multiset conditions.
#### New Rules
* The new Java rule {% rule "java/bestpractices/UseTryWithResources" %) (`java-bestpractices`) searches
for try-blocks, that could be changed to a try-with-resources statement. This statement ensures that
each resource is closed at the end of the statement and is available since Java 7.
#### Modified Rules
* The Apex rule {% rule "apex/codestyle/MethodNamingConventions" %} (apex-codestyle) has a new
property `skipTestMethodUnderscores`, which is by default disabled. The new property allows for ignoring
all test methods, either using the `testMethod` modifier or simply annotating them `@isTest`.
### Fixed Issues
* all
* [#1462](https://github.com/pmd/pmd/issues/1462): \[core] Failed build on Windows with source zip archive
* [#1559](https://github.com/pmd/pmd/issues/1559): \[core] CPD: Lexical error in file (no file name provided)
* java-bestpractices
* [#808](https://github.com/pmd/pmd/issues/808): \[java] AccessorMethodGeneration false positives with compile time constants
* [#1405](https://github.com/pmd/pmd/issues/1405): \[java] New Rule: UseTryWithResources - Replace close and IOUtils.closeQuietly with try-with-resources
* [#1555](https://github.com/pmd/pmd/issues/1555): \[java] UnusedImports false positive for method parameter type in @see Javadoc
* java-codestyle
* [#1543](https://github.com/pmd/pmd/issues/1543): \[java] LinguisticNaming should ignore overriden methods
* [#1547](https://github.com/pmd/pmd/issues/1547): \[java] AtLeastOneConstructorRule: false-positive with lombok.AllArgsConstructor
* [#1624](https://github.com/pmd/pmd/issues/1624): \[java] UseDiamondOperator false positive with var initializer
* java-design
* [#1641](https://github.com/pmd/pmd/issues/1641): \[java] False-positive with Lombok and inner classes
* java-errorprone
* [#780](https://github.com/pmd/pmd/issues/780): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors
* java-multithreading
* [#1633](https://github.com/pmd/pmd/issues/1633): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat
* java-performance
* [#1632](https://github.com/pmd/pmd/issues/1632): \[java] ConsecutiveLiteralAppends false positive over catch
* plsql
* [#1587](https://github.com/pmd/pmd/issues/1587): \[plsql] ParseException with EXISTS
* [#1589](https://github.com/pmd/pmd/issues/1589): \[plsql] ParseException with subqueries in WHERE clause
* [#1590](https://github.com/pmd/pmd/issues/1590): \[plsql] ParseException when using hierarchical query clause
* [#1656](https://github.com/pmd/pmd/issues/1656): \[plsql] ParseException with analytic functions, trim and subqueries
### API Changes
### External Contributions
* [#1623](https://github.com/pmd/pmd/pull/1623): \[java] Fix lombok.AllArgsConstructor support - [Bobby Wertman](https://github.com/CasualSuperman)
* [#1625](https://github.com/pmd/pmd/pull/1625): \[java] UnusedImports false positive for method parameter type in @see Javadoc - [Shubham](https://github.com/Shubham-2k17)
* [#1628](https://github.com/pmd/pmd/pull/1628): \[java] LinguisticNaming should ignore overriden methods - [Shubham](https://github.com/Shubham-2k17)
* [#1634](https://github.com/pmd/pmd/pull/1634): \[java] BeanMembersShouldSerializeRule does not recognize lombok accessors - [Shubham](https://github.com/Shubham-2k17)
* [#1635](https://github.com/pmd/pmd/pull/1635): \[java] UnsynchronizedStaticFormatter reports commons lang FastDateFormat - [Shubham](https://github.com/Shubham-2k17)
* [#1637](https://github.com/pmd/pmd/pull/1637): \[java] Compile time constants initialized by literals avoided by AccessorMethodGenerationRule - [Shubham](https://github.com/Shubham-2k17)
* [#1640](https://github.com/pmd/pmd/pull/1640): \[java] Update instead of override classHasLombokAnnotation flag - [Phokham Nonava](https://github.com/fluxroot)
* [#1644](https://github.com/pmd/pmd/pull/1644): \[apex] Add property to allow apex test methods to contain underscores - [Tom](https://github.com/tomdaly)
* [#1645](https://github.com/pmd/pmd/pull/1645): \[java] ConsecutiveLiteralAppends false positive - [Shubham](https://github.com/Shubham-2k17)
* [#1646](https://github.com/pmd/pmd/pull/1646): \[java] UseDiamondOperator doesn't work with var - [Shubham](https://github.com/Shubham-2k17)
* [#1654](https://github.com/pmd/pmd/pull/1654): \[core] Antlr token filter - [Tomi De Lucca](https://github.com/tomidelucca)
* [#1655](https://github.com/pmd/pmd/pull/1655): \[kotlin] Kotlin tokenizer refactor - [Lucas Soncini](https://github.com/lsoncini)
{% endtocmaker %}

View File

@ -5,15 +5,26 @@
package net.sourceforge.pmd.lang.apex.rule.codestyle;
import static apex.jorje.semantic.symbol.type.ModifierTypeInfos.OVERRIDE;
import static net.sourceforge.pmd.properties.PropertyFactory.booleanProperty;
import net.sourceforge.pmd.lang.apex.ast.ASTMethod;
import net.sourceforge.pmd.lang.apex.ast.ASTModifierNode;
import net.sourceforge.pmd.lang.apex.ast.ASTProperty;
import net.sourceforge.pmd.lang.apex.ast.ASTUserClass;
import net.sourceforge.pmd.lang.apex.rule.AbstractApexRule;
import net.sourceforge.pmd.properties.PropertyDescriptor;
public class MethodNamingConventionsRule extends AbstractApexRule {
private static final PropertyDescriptor<Boolean> SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR
= booleanProperty("skipTestMethodUnderscores")
.desc("Skip underscores in test methods")
.defaultValue(false)
.build();
public MethodNamingConventionsRule() {
definePropertyDescriptor(SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR);
setProperty(CODECLIMATE_CATEGORIES, "Style");
// Note: x10 as Apex has not automatic refactoring
setProperty(CODECLIMATE_REMEDIATION_MULTIPLIER, 1);
@ -30,6 +41,9 @@ public class MethodNamingConventionsRule extends AbstractApexRule {
if (isOverriddenMethod(node) || isPropertyAccessor(node) || isConstructor(node)) {
return data;
}
if (isTestMethod(node) && getProperty(SKIP_TEST_METHOD_UNDERSCORES_DESCRIPTOR)) {
return data;
}
String methodName = node.getImage();
@ -53,4 +67,9 @@ public class MethodNamingConventionsRule extends AbstractApexRule {
private boolean isConstructor(ASTMethod node) {
return node.getNode().getMethodInfo().isConstructor();
}
private boolean isTestMethod(ASTMethod node) {
final ASTModifierNode modifierNode = node.getFirstChildOfType(ASTModifierNode.class);
return modifierNode != null && modifierNode.getNode().getModifiers().isTest();
}
}

View File

@ -66,4 +66,38 @@ public class Foo {
]]></code>
</test-code>
<test-code>
<description>#1573 method names should not contain underscores, but skip test methods 1</description>
<expected-problems>0</expected-problems>
<rule-property name="skipTestMethodUnderscores">true</rule-property>
<code><![CDATA[
public class Foo {
@isTest
void test_barFoo() {}
}
]]></code>
</test-code>
<test-code>
<description>#1573 method names should not contain underscores, but skip test methods 2</description>
<expected-problems>0</expected-problems>
<rule-property name="skipTestMethodUnderscores">true</rule-property>
<code><![CDATA[
public class Foo {
void testMethod test_barFoo() {}
}
]]></code>
</test-code>
<test-code>
<description>#1573 method names should not contain underscores, but skip test methods 3</description>
<expected-problems>1</expected-problems>
<rule-property name="skipTestMethodUnderscores">false</rule-property>
<code><![CDATA[
public class Foo {
@isTest
void test_barFoo() {}
}
]]></code>
</test-code>
</test-data>

View File

@ -316,7 +316,7 @@ public class PMD {
// Make sure the cache is listening for analysis results
ctx.getReport().addListener(configuration.getAnalysisCache());
final RuleSetFactory silentFactoy = new RuleSetFactory(ruleSetFactory, false);
final RuleSetFactory silentFactory = new RuleSetFactory(ruleSetFactory, false);
/*
* Check if multithreaded support is available. ExecutorService can also
@ -324,9 +324,9 @@ public class PMD {
* "-threads 0" command line option.
*/
if (configuration.getThreads() > 0) {
new MultiThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers);
new MultiThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers);
} else {
new MonoThreadProcessor(configuration).processFiles(silentFactoy, files, ctx, renderers);
new MonoThreadProcessor(configuration).processFiles(silentFactory, files, ctx, renderers);
}
// Persist the analysis cache

View File

@ -6,15 +6,20 @@ package net.sourceforge.pmd.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.Token;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.cpd.token.AntlrToken;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Generic implementation of a {@link Tokenizer} useful to any Antlr grammar.
*
* @deprecated This is an internal API.
*/
@Deprecated
@InternalApi
public abstract class AntlrTokenizer implements Tokenizer {
protected abstract AntlrTokenManager getLexerForSource(SourceCode sourceCode);
@ -22,19 +27,16 @@ public abstract class AntlrTokenizer implements Tokenizer {
@Override
public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) {
AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
final AntlrTokenManager tokenManager = getLexerForSource(sourceCode);
tokenManager.setFileName(sourceCode.getFileName());
final AntlrTokenFilter tokenFilter = getTokenFilter(tokenManager);
try {
AntlrToken token = (AntlrToken) tokenManager.getNextToken();
while (token.getType() != Token.EOF) {
if (!token.isHidden()) {
final TokenEntry tokenEntry =
new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine());
tokenEntries.add(tokenEntry);
}
token = (AntlrToken) tokenManager.getNextToken();
AntlrToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
processToken(tokenEntries, tokenManager.getFileName(), currentToken);
currentToken = tokenFilter.getNextToken();
}
} catch (final AntlrTokenManager.ANTLRSyntaxError err) {
// Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled
@ -47,8 +49,17 @@ public abstract class AntlrTokenizer implements Tokenizer {
}
}
protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
return new AntlrTokenFilter(tokenManager);
}
/* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
return CharStreams.fromString(buffer.toString());
}
private void processToken(final Tokens tokenEntries, final String fileName, final AntlrToken token) {
final TokenEntry tokenEntry = new TokenEntry(token.getImage(), fileName, token.getBeginLine());
tokenEntries.add(tokenEntry);
}
}

View File

@ -0,0 +1,46 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.internal;
import java.io.IOException;
import net.sourceforge.pmd.cpd.SourceCode;
import net.sourceforge.pmd.cpd.TokenEntry;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
public abstract class JavaCCTokenizer implements Tokenizer {
protected abstract TokenManager getLexerForSource(SourceCode sourceCode);
protected TokenFilter getTokenFilter(TokenManager tokenManager) {
return new JavaCCTokenFilter(tokenManager);
}
protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) {
return new TokenEntry(currentToken.getImage(), filename, currentToken.getBeginLine());
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
TokenManager tokenManager = getLexerForSource(sourceCode);
tokenManager.setFileName(sourceCode.getFileName());
try {
final TokenFilter tokenFilter = getTokenFilter(tokenManager);
GenericToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName()));
currentToken = tokenFilter.getNextToken();
}
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
}
}

View File

@ -71,4 +71,8 @@ public class AntlrToken implements GenericToken {
public boolean isHidden() {
return token.getChannel() == Lexer.HIDDEN;
}
public boolean isDefault() {
return token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL;
}
}

View File

@ -0,0 +1,30 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token;
import static org.antlr.v4.runtime.Token.EOF;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
/**
* A generic filter for Antlr-based token managers that allows to use comments
* to enable / disable analysis of parts of the stream
*/
public class AntlrTokenFilter extends BaseTokenFilter<AntlrToken> {
/**
* Creates a new AntlrTokenFilter
* @param tokenManager The token manager from which to retrieve tokens to be filtered
*/
public AntlrTokenFilter(final AntlrTokenManager tokenManager) {
super(tokenManager);
}
@Override
protected boolean shouldStopProcessing(final AntlrToken currentToken) {
return currentToken.getType() == EOF;
}
}

View File

@ -4,6 +4,7 @@
package net.sourceforge.pmd.cpd.token;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@ -11,74 +12,18 @@ import net.sourceforge.pmd.lang.ast.GenericToken;
* A generic filter for JavaCC-based token managers that allows to use comments
* to enable / disable analysis of parts of the stream
*/
public class JavaCCTokenFilter implements TokenFilter {
public class JavaCCTokenFilter extends BaseTokenFilter<GenericToken> {
private final TokenManager tokenManager;
private boolean discardingSuppressing;
/**
* Creates a new JavaCCTokenFilter
* @param tokenManager The token manager from which to retrieve tokens to be filtered
*/
public JavaCCTokenFilter(final TokenManager tokenManager) {
this.tokenManager = tokenManager;
super(tokenManager);
}
@Override
public final GenericToken getNextToken() {
GenericToken currentToken = (GenericToken) tokenManager.getNextToken();
while (!currentToken.getImage().isEmpty()) {
analyzeToken(currentToken);
processCPDSuppression(currentToken);
if (!isDiscarding()) {
return currentToken;
}
currentToken = (GenericToken) tokenManager.getNextToken();
}
return null;
protected boolean shouldStopProcessing(final GenericToken currentToken) {
return currentToken.getImage().isEmpty();
}
private boolean isDiscarding() {
return discardingSuppressing || isLanguageSpecificDiscarding();
}
private void processCPDSuppression(final GenericToken currentToken) {
// Check if a comment is altering the suppression state
GenericToken comment = currentToken.getPreviousComment();
while (comment != null) {
if (comment.getImage().contains("CPD-OFF")) {
discardingSuppressing = true;
break;
}
if (comment.getImage().contains("CPD-ON")) {
discardingSuppressing = false;
break;
}
comment = comment.getPreviousComment();
}
}
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
* @return True if tokens should be filtered, false otherwise
*/
protected boolean isLanguageSpecificDiscarding() {
return false;
}
/**
* Extension point for subclasses to analyze all tokens (before filtering)
* and update internal status to decide on custom discard rules.
*
* @param currentToken The token to be analyzed
* @see #isLanguageSpecificDiscarding()
*/
protected void analyzeToken(final GenericToken currentToken) {
// noop
}
}

View File

@ -0,0 +1,93 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token.internal;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
/**
* A generic filter for PMD token managers that allows to use comments
* to enable / disable analysis of parts of the stream
*/
public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFilter {
private final TokenManager tokenManager;
private boolean discardingSuppressing;
/**
* Creates a new BaseTokenFilter
* @param tokenManager The token manager from which to retrieve tokens to be filtered
*/
public BaseTokenFilter(final TokenManager tokenManager) {
this.tokenManager = tokenManager;
}
@Override
public final T getNextToken() {
T currentToken = (T) tokenManager.getNextToken();
while (!shouldStopProcessing(currentToken)) {
analyzeToken(currentToken);
processCPDSuppression(currentToken);
if (!isDiscarding()) {
return currentToken;
}
currentToken = (T) tokenManager.getNextToken();
}
return null;
}
private boolean isDiscarding() {
return discardingSuppressing || isLanguageSpecificDiscarding();
}
private void processCPDSuppression(final T currentToken) {
// Check if a comment is altering the suppression state
GenericToken comment = currentToken.getPreviousComment();
while (comment != null) {
if (comment.getImage().contains("CPD-OFF")) {
discardingSuppressing = true;
break;
}
if (comment.getImage().contains("CPD-ON")) {
discardingSuppressing = false;
break;
}
comment = comment.getPreviousComment();
}
}
/**
* Extension point for subclasses to analyze all tokens (before filtering)
* and update internal status to decide on custom discard rules.
*
* @param currentToken The token to be analyzed
* @see #isLanguageSpecificDiscarding()
*/
protected void analyzeToken(final T currentToken) {
// noop
}
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
* @return True if tokens should be filtered, false otherwise
*/
protected boolean isLanguageSpecificDiscarding() {
return false;
}
/**
* Extension point for subclasses to indicate when to stop filtering tokens.
*
* @param currentToken The token to be analyzed
* @return True if the token filter has finished consuming all tokens, false otherwise
*/
protected abstract boolean shouldStopProcessing(T currentToken);
}

View File

@ -34,6 +34,14 @@ public class AntlrTokenManager implements TokenManager {
@Override
public Object getNextToken() {
AntlrToken nextToken = getNextTokenFromAnyChannel();
while (!nextToken.isDefault()) {
nextToken = getNextTokenFromAnyChannel();
}
return nextToken;
}
private AntlrToken getNextTokenFromAnyChannel() {
final AntlrToken previousComment = previousToken != null && previousToken.isHidden() ? previousToken : null;
final AntlrToken currentToken = new AntlrToken(lexer.nextToken(), previousComment);
previousToken = currentToken;

View File

@ -0,0 +1,14 @@
<?xml version="1.0"?>
<ruleset name="6120"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
<description>
This ruleset contains links to rules that are new in PMD v6.12.0
</description>
<rule ref="category/java/bestpractices.xml/UseTryWithResources"/>
</ruleset>

View File

@ -12,6 +12,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.junit.After;
@ -104,13 +105,18 @@ public class YAHTMLRendererTest extends AbstractRendererTst {
try (FileInputStream in = new FileInputStream(new File(outputDir, file));
InputStream expectedIn = YAHTMLRendererTest.class.getResourceAsStream("yahtml/" + file)) {
String data = IOUtils.toString(in, StandardCharsets.UTF_8);
String expected = IOUtils.toString(expectedIn, StandardCharsets.UTF_8);
String expected = normalizeLineSeparators(IOUtils.toString(expectedIn, StandardCharsets.UTF_8));
assertEquals("File " + file + " is different", expected, data);
}
}
}
private static String normalizeLineSeparators(String s) {
return s.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_WINDOWS), IOUtils.LINE_SEPARATOR_UNIX)
.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_UNIX), PMD.EOL);
}
@Override
public Renderer getRenderer() {
Renderer result = new YAHTMLRenderer();

View File

@ -6,22 +6,19 @@ package net.sourceforge.pmd.cpd;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Properties;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.cpp.CppTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**
* The C++ tokenizer.
*/
public class CPPTokenizer implements Tokenizer {
public class CPPTokenizer extends JavaCCTokenizer {
private boolean skipBlocks = true;
private String skipBlocksStart;
@ -49,26 +46,6 @@ public class CPPTokenizer implements Tokenizer {
}
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) {
final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader));
GenericToken currentToken = tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
currentToken = tokenFilter.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError | IOException err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
}
}
private String maybeSkipBlocks(String test) throws IOException {
if (!skipBlocks) {
return test;
@ -92,4 +69,14 @@ public class CPPTokenizer implements Tokenizer {
}
return filtered.toString();
}
@Override
protected TokenManager getLexerForSource(SourceCode sourceCode) {
try {
StringBuilder buffer = sourceCode.getCodeBuffer();
return new CppTokenManager(IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString()))));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@ -117,7 +118,7 @@ public class CPPTokenizerContinuationTest {
.getResourceAsStream("cpp/" + name), StandardCharsets.UTF_8);
}
private Tokens parse(String code) {
private Tokens parse(String code) throws IOException {
CPPTokenizer tokenizer = new CPPTokenizer();
tokenizer.setProperties(new Properties());
Tokens tokens = new Tokens();

View File

@ -8,16 +8,23 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class CPPTokenizerTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testUTFwithBOM() {
Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
@ -69,21 +76,31 @@ public class CPPTokenizerTest {
@Test
public void testTokenizerWithSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, true);
Tokens tokens = parse(test, true, new Tokens());
assertEquals(19, tokens.size());
}
@Test
public void testTokenizerWithSkipBlocksPattern() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, true, "#if debug|#endif");
Tokens tokens = new Tokens();
try {
parse(test, true, "#if debug|#endif", tokens);
} catch (TokenMgrError ignored) {
// ignored
}
assertEquals(31, tokens.size());
}
@Test
public void testTokenizerWithoutSkipBlocks() throws Exception {
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"), StandardCharsets.UTF_8);
Tokens tokens = parse(test, false);
Tokens tokens = new Tokens();
try {
parse(test, false, tokens);
} catch (TokenMgrError ignored) {
// ignored
}
assertEquals(37, tokens.size());
}
@ -128,15 +145,33 @@ public class CPPTokenizerTest {
assertEquals(9, tokens.size());
}
@Test
public void testLexicalErrorFilename() throws Exception {
Properties properties = new Properties();
properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(false));
String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/issue-1559.cpp"), StandardCharsets.UTF_8);
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(test, "issue-1559.cpp"));
CPPTokenizer tokenizer = new CPPTokenizer();
tokenizer.setProperties(properties);
expectedException.expect(TokenMgrError.class);
expectedException.expectMessage("Lexical error in file issue-1559.cpp at");
tokenizer.tokenize(code, new Tokens());
}
private Tokens parse(String snippet) {
return parse(snippet, false);
try {
return parse(snippet, false, new Tokens());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Tokens parse(String snippet, boolean skipBlocks) {
return parse(snippet, skipBlocks, null);
private Tokens parse(String snippet, boolean skipBlocks, Tokens tokens) throws IOException {
return parse(snippet, skipBlocks, null, tokens);
}
private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
private Tokens parse(String snippet, boolean skipBlocks, String skipPattern, Tokens tokens) throws IOException {
Properties properties = new Properties();
properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
if (skipPattern != null) {
@ -147,7 +182,6 @@ public class CPPTokenizerTest {
tokenizer.setProperties(properties);
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
Tokens tokens = new Tokens();
tokenizer.tokenize(code, tokens);
return tokens;
}

View File

@ -0,0 +1,11 @@
namespace ABC
{
namespace DEF
{
#ifdef USE_QT
const char* perPixelQml = R"QML(
)QML";
}
}
#endif // USE_QT

View File

@ -8,9 +8,13 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import net.sourceforge.pmd.PMD;
public class MockedFileWriter implements FileWriter {
public static class FileEntry {
@ -43,4 +47,9 @@ public class MockedFileWriter implements FileWriter {
public void reset() {
data.clear();
}
public static String normalizeLineSeparators(String s) {
return s.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_WINDOWS), IOUtils.LINE_SEPARATOR_UNIX)
.replaceAll(Pattern.quote(IOUtils.LINE_SEPARATOR_UNIX), PMD.EOL);
}
}

View File

@ -67,6 +67,11 @@ public class RuleDocGeneratorTest {
});
}
private static String loadResource(String name) throws IOException {
return MockedFileWriter.normalizeLineSeparators(
IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream(name), StandardCharsets.UTF_8));
}
@Test
public void testSingleRuleset() throws RuleSetNotFoundException, IOException {
RuleDocGenerator generator = new RuleDocGenerator(writer, root);
@ -82,17 +87,14 @@ public class RuleDocGeneratorTest {
assertEquals(3, writer.getData().size());
FileEntry languageIndex = writer.getData().get(0);
assertTrue(FilenameUtils.normalize(languageIndex.getFilename(), true).endsWith("docs/pages/pmd/rules/java.md"));
assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/java.md"), StandardCharsets.UTF_8),
languageIndex.getContent());
assertEquals(loadResource("/expected/java.md"), languageIndex.getContent());
FileEntry ruleSetIndex = writer.getData().get(1);
assertTrue(FilenameUtils.normalize(ruleSetIndex.getFilename(), true).endsWith("docs/pages/pmd/rules/java/sample.md"));
assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/sample.md"), StandardCharsets.UTF_8),
ruleSetIndex.getContent());
assertEquals(loadResource("/expected/sample.md"), ruleSetIndex.getContent());
FileEntry sidebar = writer.getData().get(2);
assertTrue(FilenameUtils.normalize(sidebar.getFilename(), true).endsWith("docs/_data/sidebars/pmd_sidebar.yml"));
assertEquals(IOUtils.toString(RuleDocGeneratorTest.class.getResourceAsStream("/expected/pmd_sidebar.yml"), StandardCharsets.UTF_8),
sidebar.getContent());
assertEquals(loadResource("/expected/pmd_sidebar.yml"), sidebar.getContent());
}
}

View File

@ -55,6 +55,8 @@ public class SidebarGeneratorTest {
}
String yaml = new Yaml(options).dump(result);
assertEquals(IOUtils.toString(SidebarGeneratorTest.class.getResourceAsStream("sidebar.yml"), StandardCharsets.UTF_8), yaml);
String expected = MockedFileWriter.normalizeLineSeparators(
IOUtils.toString(SidebarGeneratorTest.class.getResourceAsStream("sidebar.yml"), StandardCharsets.UTF_8));
assertEquals(expected, yaml);
}
}

View File

@ -0,0 +1,36 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
public class GoCPDTokenizerTest extends AbstractTokenizerTest {
private static final String FILENAME = "hello.go";
@Before
@Override
public void buildTokenizer() throws IOException {
this.tokenizer = new GoTokenizer();
this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
}
@Override
public String getSampleCode() throws IOException {
return IOUtils.toString(GoTokenizer.class.getResourceAsStream(FILENAME));
}
@Test
public void tokenizeTest() throws IOException {
this.expectedTokenCount = 23;
super.tokenizeTest();
}
}

Some files were not shown because too many files have changed in this diff Show More