From f268377c4dd7339c89cf2013a295ed82af5a949d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 3 Apr 2019 01:01:33 -0300 Subject: [PATCH 01/23] [go] Fix grammar around escape characters - Fixes #1751 --- .../sourceforge/pmd/lang/go/antlr4/Golang.g4 | 4 +-- .../pmd/cpd/EdgeCasesTokenizerTest.java | 25 +++++++++++++++++++ .../net/sourceforge/pmd/cpd/issue1751.go | 6 +++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java create mode 100644 pmd-go/src/test/resources/net/sourceforge/pmd/cpd/issue1751.go diff --git a/pmd-go/src/main/antlr4/net/sourceforge/pmd/lang/go/antlr4/Golang.g4 b/pmd-go/src/main/antlr4/net/sourceforge/pmd/lang/go/antlr4/Golang.g4 index ec54cbac43..673c0b6682 100644 --- a/pmd-go/src/main/antlr4/net/sourceforge/pmd/lang/go/antlr4/Golang.g4 +++ b/pmd-go/src/main/antlr4/net/sourceforge/pmd/lang/go/antlr4/Golang.g4 @@ -917,10 +917,10 @@ RUNE_LIT //unicode_value = unicode_char | little_u_value | big_u_value | escaped_char . fragment UNICODE_VALUE - : UNICODE_CHAR - | LITTLE_U_VALUE + : LITTLE_U_VALUE | BIG_U_VALUE | ESCAPED_CHAR + | UNICODE_CHAR ; //byte_value = octal_byte_value | hex_byte_value . diff --git a/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java new file mode 100644 index 0000000000..e3ca7d0e58 --- /dev/null +++ b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java @@ -0,0 +1,25 @@ + +package net.sourceforge.pmd.cpd; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +public class EdgeCasesTokenizerTest { + + private String getSampleCode(final String filename) throws IOException { + return IOUtils.toString(GoTokenizer.class.getResourceAsStream(filename)); + } + + @Test + public void testEscapedBackSlash() throws IOException { + // See https://github.com/pmd/pmd/issues/1751 + final String filename = "issue1751.go"; + final GoTokenizer tokenizer = new GoTokenizer(); + final SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(getSampleCode(filename), filename)); + + final Tokens tokens = new Tokens(); + tokenizer.tokenize(sourceCode, tokens); // it should simply not fail + } +} diff --git a/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/issue1751.go b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/issue1751.go new file mode 100644 index 0000000000..1e0042e68a --- /dev/null +++ b/pmd-go/src/test/resources/net/sourceforge/pmd/cpd/issue1751.go @@ -0,0 +1,6 @@ +func test(in *Value, param *Value) (*Value, *Error) { + output := strings.Replace(in.String(), "\\", "\\\\", -1) + output = strings.Replace(output, "\"", "\\\"", -1) + output = strings.Replace(output, "'", "\\'", -1) + return AsValue(output), nil +} From 3aea3cfe2979491b35a562d8c50896065cbb5570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 3 Apr 2019 01:02:54 -0300 Subject: [PATCH 02/23] Update changelog, refs #1751 --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 8b53655c31..5e8fe25beb 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,6 +16,8 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* go + * [#1751](https://github.com/pmd/pmd/issues/1751): \[go] Parsing errors encountered with escaped backslash * java * [#1729](https://github.com/pmd/pmd/issues/1729): \[java] JavaRuleViolation loses information in `className` field when class has package-private access level * java-bestpractices From 06625b23f966edf928e81c8fa15b9e2f780392e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Kautler?= Date: Wed, 3 Apr 2019 00:53:10 +0200 Subject: [PATCH 03/23] Fix UseObjectForClearerAPI rule that checked non-public methods --- docs/pages/pmd/rules/java/design.md | 2 +- .../main/resources/category/java/design.xml | 2 +- .../design/xml/UseObjectForClearerAPI.xml | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/pages/pmd/rules/java/design.md b/docs/pages/pmd/rules/java/design.md index 6ddb0d0ae2..ec3d56e442 100644 --- a/docs/pages/pmd/rules/java/design.md +++ b/docs/pages/pmd/rules/java/design.md @@ -2090,7 +2090,7 @@ your API. **This rule is defined by the following XPath expression:** ``` xpath -//MethodDeclaration[@Public]/MethodDeclarator/FormalParameters[ +//MethodDeclaration[@Public = 'true']/MethodDeclarator/FormalParameters[ count(FormalParameter/Type/ReferenceType/ClassOrInterfaceType[@Image = 'String']) > 3 ] ``` diff --git a/pmd-java/src/main/resources/category/java/design.xml b/pmd-java/src/main/resources/category/java/design.xml index c2be0c822d..a6d1d06f69 100644 --- a/pmd-java/src/main/resources/category/java/design.xml +++ b/pmd-java/src/main/resources/category/java/design.xml @@ -1638,7 +1638,7 @@ your API. 3 ] ]]> diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml index 56aa3d40e4..a969d3e65e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml @@ -58,4 +58,36 @@ public class MyClass { ]]> + + + + + 0 + + + + From f3c0d39f66ce2bf999940623854a3399dfffc15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Kautler?= Date: Wed, 3 Apr 2019 12:00:40 +0200 Subject: [PATCH 04/23] Fix UseObjectForClearerAPI rule that treated String and String[] the same --- .../main/resources/category/java/design.xml | 2 +- .../design/xml/UseObjectForClearerAPI.xml | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pmd-java/src/main/resources/category/java/design.xml b/pmd-java/src/main/resources/category/java/design.xml index a6d1d06f69..41cc421ca4 100644 --- a/pmd-java/src/main/resources/category/java/design.xml +++ b/pmd-java/src/main/resources/category/java/design.xml @@ -1639,7 +1639,7 @@ your API. 3 + count(FormalParameter/Type/ReferenceType/ClassOrInterfaceType[@Image = 'String' and @Array = 'false']) > 3 ] ]]> diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml index a969d3e65e..c8989343d9 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml @@ -86,6 +86,26 @@ public class MyClass { String databaseAdress) { } +} + ]]> + + + + + + + 0 + + From 2e8056dc2877d6499f2d614234f59ef3e62fc8b1 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 5 Apr 2019 14:10:57 +0200 Subject: [PATCH 05/23] Fix checkstyle --- .../net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java index e3ca7d0e58..1d56e989f0 100644 --- a/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java +++ b/pmd-go/src/test/java/net/sourceforge/pmd/cpd/EdgeCasesTokenizerTest.java @@ -1,7 +1,11 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ package net.sourceforge.pmd.cpd; import java.io.IOException; +import java.nio.charset.StandardCharsets; import org.apache.commons.io.IOUtils; import org.junit.Test; @@ -9,9 +13,9 @@ import org.junit.Test; public class EdgeCasesTokenizerTest { private String getSampleCode(final String filename) throws IOException { - return IOUtils.toString(GoTokenizer.class.getResourceAsStream(filename)); + return IOUtils.toString(GoTokenizer.class.getResourceAsStream(filename), StandardCharsets.UTF_8); } - + @Test public void testEscapedBackSlash() throws IOException { // See https://github.com/pmd/pmd/issues/1751 From 96a14ede0aae28dbc289f59f7a7190dd8ca4872a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 5 Apr 2019 14:44:45 +0200 Subject: [PATCH 06/23] Simplify test xml CDATA is not necessary for test case descriptions --- .../design/xml/UseObjectForClearerAPI.xml | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml index c8989343d9..0cc0eeb476 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/design/xml/UseObjectForClearerAPI.xml @@ -4,12 +4,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pmd.sourceforge.net/rule-tests http://pmd.sourceforge.net/rule-tests_1_0_0.xsd"> - + No issues here 0 - - + ]]> + - + Public method with 4 String parameters 1 - - + ]]> + - - - + non-public methods should not have findings 0 - - - + String[] should not be treated as String 0 Date: Fri, 5 Apr 2019 14:45:03 +0200 Subject: [PATCH 07/23] Update release notes, fixes #1760, refs #1752 --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 8b53655c31..a3a74a6ed3 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -20,6 +20,8 @@ This is a {{ site.pmd.release_type }} release. * [#1729](https://github.com/pmd/pmd/issues/1729): \[java] JavaRuleViolation loses information in `className` field when class has package-private access level * java-bestpractices * [#1720](https://github.com/pmd/pmd/issues/1720): \[java] UnusedImports false positive for Javadoc link with array type +* java-design + * [#1760](https://github.com/pmd/pmd/issues/1760): \[java] UseObjectForClearerAPI flags private methods ### API Changes @@ -27,6 +29,7 @@ This is a {{ site.pmd.release_type }} release. * [#1745](https://github.com/pmd/pmd/pull/1745): \[doc] Fixed some errors in docs - [0xflotus](https://github.com/0xflotus) * [#1746](https://github.com/pmd/pmd/pull/1746): \[java] Update rule to prevent UnusedImport when using JavaDoc with array type - [itaigilo](https://github.com/itaigilo) +* [#1752](https://github.com/pmd/pmd/pull/1752): \[java] UseObjectForClearerAPI Only For Public - [Björn Kautler](https://github.com/Vampire) {% endtocmaker %} From 3bf43b3f8e21e65ac512a001e157fd5c8cfa5d8e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 5 Apr 2019 14:53:53 +0200 Subject: [PATCH 08/23] [doc] Update doc --- docs/pages/pmd/rules/java/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/pmd/rules/java/design.md b/docs/pages/pmd/rules/java/design.md index ec3d56e442..b0647c24cd 100644 --- a/docs/pages/pmd/rules/java/design.md +++ b/docs/pages/pmd/rules/java/design.md @@ -2091,7 +2091,7 @@ your API. **This rule is defined by the following XPath expression:** ``` xpath //MethodDeclaration[@Public = 'true']/MethodDeclarator/FormalParameters[ - count(FormalParameter/Type/ReferenceType/ClassOrInterfaceType[@Image = 'String']) > 3 + count(FormalParameter/Type/ReferenceType/ClassOrInterfaceType[@Image = 'String' and @Array = 'false']) > 3 ] ``` From 8db2026735d578955b00be9b13ecd0741e0b510d Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Fri, 5 Apr 2019 15:19:28 +0200 Subject: [PATCH 09/23] Added Dart support to CPD. The tokenizer uses an ANTLR4 grammar based on the one at https://github.com/antlr/grammars-v4/tree/master/dart2. --- pmd-dart/pom.xml | 57 + .../sourceforge/pmd/lang/dart/antlr4/Dart2.g4 | 995 ++++++++++++++++++ .../net/sourceforge/pmd/cpd/DartLanguage.java | 18 + .../sourceforge/pmd/cpd/DartTokenizer.java | 70 ++ .../services/net.sourceforge.pmd.cpd.Language | 1 + pmd-dist/pom.xml | 5 + pom.xml | 1 + 7 files changed, 1147 insertions(+) create mode 100644 pmd-dart/pom.xml create mode 100644 pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 create mode 100644 pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartLanguage.java create mode 100644 pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java create mode 100644 pmd-dart/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language diff --git a/pmd-dart/pom.xml b/pmd-dart/pom.xml new file mode 100644 index 0000000000..b0295ebf93 --- /dev/null +++ b/pmd-dart/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + pmd-dart + PMD Dart + + + net.sourceforge.pmd + pmd + 6.14.0-SNAPSHOT + + + + + + org.antlr + antlr4-maven-plugin + + + + maven-resources-plugin + + false + + ${*} + + + + + + + + + org.antlr + antlr4-runtime + + + net.sourceforge.pmd + pmd-core + + + commons-io + commons-io + + + + junit + junit + test + + + net.sourceforge.pmd + pmd-test + test + + + diff --git a/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 b/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 new file mode 100644 index 0000000000..78d3083d2c --- /dev/null +++ b/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 @@ -0,0 +1,995 @@ +/* + [The "BSD licence"] + Copyright (c) 2019 Wener + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +grammar Dart2; + +compilationUnit: libraryDefinition | partDeclaration; + +WHITESPACE +// : ('\t' | ' ' | NEWLINE)+ -> skip + : [ \t\r\n\u000C]+ -> skip + ; + +// 8 Variables +variableDeclaration + : declaredIdentifier (',' identifier)* + ; + +declaredIdentifier + : metadata finalConstVarOrType identifier + ; +finalConstVarOrType + : 'final' type? + | 'const' type? + | varOrType + ; +varOrType + : 'var' + | type + ; + +initializedVariableDeclaration + : declaredIdentifier ('=' expression)? (','initializedIdentifier)* + ; +initializedIdentifier + : identifier ('=' expression)? + ; +initializedIdentifierList + : initializedIdentifier (',' initializedIdentifier)* + ; + + + + +// 9 Functions +functionSignature + : metadata returnType? identifier formalParameterPart + ; +formalParameterPart + : typeParameters? formalParameterList + ; +returnType + : 'void' + | type + ; + +functionBody + : 'async'? '=>' expression ';' + | ('async' | 'async*' | 'sync*')? block + ; +block + : '{' statements '}' + ; + +// 9.2 Formal Parameters +formalParameterList + : '(' ')' + | '(' normalFormalParameters ')' + | '(' normalFormalParameters (',' optionalFormalParameters)? ')' + | '(' optionalFormalParameters ')' + ; +normalFormalParameters + : normalFormalParameter (',' normalFormalParameter)* + ; +optionalFormalParameters + : optionalPositionalFormalParameters + | namedFormalParameters + ; +optionalPositionalFormalParameters + : '[' defaultFormalParameter (',' defaultFormalParameter)* ','? ']' + ; +namedFormalParameters + : '{' defaultNamedParameter (',' defaultNamedParameter)* ','? '}' + ; + +// 9.2.1 Required Formals +normalFormalParameter + : functionFormalParameter + | fieldFormalParameter + | simpleFormalParameter + ; +functionFormalParameter + : metadata 'covariant'? returnType? identifier formalParameterPart + ; +simpleFormalParameter + : declaredIdentifier + | metadata 'covariant'? identifier + ; +fieldFormalParameter + : metadata finalConstVarOrType? 'this' '.' identifier formalParameterPart? + ; + +// 9.2.2 Optional Formals +defaultFormalParameter + : normalFormalParameter ('=' expression)? + ; +defaultNamedParameter + : normalFormalParameter ('=' expression)? + | normalFormalParameter (':' expression)? + ; + +// 10 Classes +classDefinition + : metadata 'abstract'? 'class' identifier typeParameters? + superclass? mixins? interfaces? + '{' (metadata classMemberDefinition)* '}' + | metadata 'abstract'? 'class' mixinApplicationClass +; +mixins + : 'with' typeList + ; +classMemberDefinition + : declaration ';' + | methodSignature functionBody + ; +methodSignature + : constructorSignature initializers? + | factoryConstructorSignature + | 'static'? functionSignature + | 'static'? getterSignature + | 'static'? setterSignature + | operatorSignature + ; + + +declaration + : constantConstructorSignature (redirection | initializers)? + | constructorSignature (redirection | initializers)? + | 'external' constantConstructorSignature + | 'external' constructorSignature + | ('external' 'static'?)? getterSignature + | ('external' 'static'?)? setterSignature + | 'external'? operatorSignature + | ('external' 'static'?)? functionSignature + | 'static' ('final' | 'const') type? staticFinalDeclarationList + | 'final' type? initializedIdentifierList + | ('static' | 'covariant')? ('var' | type) initializedIdentifierList + ; + +staticFinalDeclarationList + : staticFinalDeclaration (',' staticFinalDeclaration)* + ; +staticFinalDeclaration + : identifier '=' expression + ; + +// 10.1.1 Operators +operatorSignature + : returnType? 'operator' operator formalParameterList + ; +operator + : '~' | binaryOperator | '[]' | '[]=' + ; + +binaryOperator + : multiplicativeOperator + | additiveOperator + | shiftOperator + | relationalOperator + | '==' + | bitwiseOperator + ; +// 10.2 Getters +getterSignature + : returnType? 'get' identifier + ; +// 10.2 Setters +setterSignature + : returnType? 'set' identifier formalParameterList + ; + +// 10.6 Constructors +constructorSignature + : identifier ('.' identifier)? formalParameterList + ; +redirection + : ':' 'this' ('.' identifier)? arguments + ; + +initializers + : ':' initializerListEntry (',' initializerListEntry)* + ; +initializerListEntry + : 'super' arguments + | 'super' '.' identifier arguments + | fieldInitializer + | assertion + ; +fieldInitializer + : ('this' '.')? identifier '=' conditionalExpression cascadeSection* + ; + +// 10.6.2 Factories +factoryConstructorSignature + : 'factory' identifier ('.' identifier)? formalParameterList + ; +redirectingFactoryConstructorSignature + : 'const'? 'factory' identifier ('.' identifier)? formalParameterList '=' + type ('.' identifier)? + ; +// 10.6.3 Constant Constructors +constantConstructorSignature: 'const' qualified formalParameterList; + +// 10.9 Supperclasses +superclass: 'extends' type; + +// 10.10 SUperinterfaces +interfaces: 'implements' typeList; + +// 12.1 Mixin Application +mixinApplicationClass + : identifier typeParameters? '=' mixinApplication ';'; +mixinApplication + : type mixins interfaces? + ; + +// 13 Enums +enumType + : metadata 'enum' identifier + '{' enumEntry (',' enumEntry)* ','? '}' + ; + +enumEntry + : metadata identifier + ; + +// 14 Generics +typeParameter + : metadata identifier ('extends' type)? + ; +typeParameters + : '<' typeParameter (',' typeParameter)* '>' + ; + +// 15 Metadata +metadata + : ('@' qualified ('.' identifier)? arguments?)* + ; + +// 16 Expressions +expression + : assignableExpression assignmentOperator expression + | conditionalExpression cascadeSection* + | throwExpression + ; +expressionWithoutCascade + : assignableExpression assignmentOperator expressionWithoutCascade + | conditionalExpression + | throwExpressionWithoutCascade + ; +expressionList + : expression (',' expression)* + ; +primary + : thisExpression + | 'super' unconditionalAssignableSelector + | functionExpression + | literal + | identifier + | newExpression + | constObjectExpression + | '(' expression ')' + ; + +// 16.1 Constants + +literal + : nullLiteral + | booleanLiteral + | numericLiteral + | stringLiteral + | symbolLiteral + | mapLiteral + | listLiteral + ; +nullLiteral: 'null'; + +numericLiteral + : NUMBER + | HEX_NUMBER + ; + +NUMBER + : DIGIT+ ('.' DIGIT+)? EXPONENT? + | '.' DIGIT+ EXPONENT? + ; +fragment +EXPONENT + : ('e' | 'E') ('+' | '-')? DIGIT+ + ; +HEX_NUMBER + : '0x' HEX_DIGIT+ + | '0X' HEX_DIGIT+ + ; +fragment +HEX_DIGIT + : [a-f] + | [A-F] + | DIGIT + ; + +booleanLiteral + : 'true' + | 'false' + ; + +//stringLiteral: (MultilineString | SingleLineString)+; +stringLiteral: SingleLineString; +//stringLiteral: SingleLineString; +SingleLineString + : '"' ~["]* '"' + | '\'' ~[']* '\'' +// | 'r\'' (~('\'' | NEWLINE))* '\'' // TODO +// | 'r"' (~('\'' | NEWLINE))* '"' + ; + +//MultilineString +// : '"""' StringContentTDQ* '"""' +// | '\'\'\'' StringContentTDQ* '\'\'\'' +// | 'r"""' (~'"""')* '"""' // TODO +// | 'r\'\'\'' (~'\'\'\'')* '\'\'\'' +// ; +//StringContentSQ: .;// TODO +//StringContentTDQ: .;// TODO +fragment +ESCAPE_SEQUENCE + : '\\n' + | '\\r' + | '\\f' + | '\\b' + | '\\t' + | '\\v' + | '\\x' HEX_DIGIT HEX_DIGIT + | '\\u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT + | '\\u{' HEX_DIGIT_SEQUENCE '}' + ; +fragment +HEX_DIGIT_SEQUENCE + : HEX_DIGIT HEX_DIGIT? HEX_DIGIT? + HEX_DIGIT? HEX_DIGIT? HEX_DIGIT? + ; +/*TODO + ::= \~{}( `\\' | `"' | `$' | ) + \alt `\\' \~{}( ) + \alt + + ::= \~{}( `\\' | `\'' | `$' | ) + \alt `\\' \~{}( ) + \alt + ::= \~{}( `\\' | `"""' | `$') + \alt `\\' \~{}( ) + \alt + + ::= \~{}( `\\' | `\'\'\'' | `$') + \alt `\\' \~{}( ) + \alt +*/ +NEWLINE + : '\n' + | '\r' + | '\r\n' + ; + +// 16.5.1 String Interpolation +stringInterpolation +// : '$' IDENTIFIER_NO_DOLLAR + : '$' identifier// FIXME + | '${' expression '}' + ; + +// 16.6 Symbols +symbolLiteral + : '#' (operator | (identifier (',' identifier)*)) + ; +// 16.7 Lists +listLiteral + : 'const'? typeArguments? '[' (expressionList ','?)? ']' + ; + +// 16.8 Maps +mapLiteral + : 'const'? typeArguments? + '{' (mapLiteralEntry (',' mapLiteralEntry)* ','?)? '}' +; +mapLiteralEntry + : expression ':' expression + ; + +// 16.9 Throw +throwExpression + : 'throw' expression + ; +throwExpressionWithoutCascade + : 'throw' expressionWithoutCascade + ; + +// 16.10 Function Expressions +functionExpression + : formalParameterPart functionBody + ; + +// 16.11 This +thisExpression: 'this'; + +// 16.12.1 New +newExpression + : 'new' type ('.' identifier)? arguments + ; + +// 16.12.2 Const +constObjectExpression + : 'const' type ('.' identifier)? arguments + ; + +// 16.14.1 Actual Argument List Evaluation +arguments + : '(' (argumentList ','?)? ')' + ; +argumentList + : namedArgument (',' namedArgument)* + | expressionList (',' namedArgument)* + ; +namedArgument + : label expression + ; + +// 16.18.2 Cascaded Invocations +cascadeSection + : '..' (cascadeSelector argumentPart*) + (assignableSelector argumentPart*)* + (assignmentOperator expressionWithoutCascade)? + ; +cascadeSelector + : '[' expression ']' + | identifier + ; +argumentPart + : typeArguments? arguments + ; + +// 16.20 Assignment +assignmentOperator + : '=' + | compoundAssignmentOperator + ; + +// 16.20.1 Compound Assignment +compoundAssignmentOperator + : '*=' + | '/=' + | '~/=' + | '%=' + | '+=' + | '<<=' + | '>>=' + | '>>>=' + | '&=' + | '^=' + | '|=' + | '??=' + ; + +// 16.21 Conditional +conditionalExpression + : ifNullExpression + ('?' expressionWithoutCascade ':' expressionWithoutCascade)? + ; +// 16.22 If-null Expression +ifNullExpression + : logicalOrExpression ('??' logicalOrExpression)* + ; + +// 16.23 Logical Boolean Expressions +logicalOrExpression + : logicalAndExpression ('||' logicalAndExpression)* + ; +logicalAndExpression + : equalityExpression ('&&' equalityExpression)* + ; + +// 16.24 Equality +equalityExpression + : relationalExpression (equalityOperator relationalExpression)? + | 'super' equalityOperator relationalExpression + ; +equalityOperator + : '==' + | '!=' + ; + +// 16.25 Relational Expressions +relationalExpression + : bitwiseOrExpression + ( + typeTest + | typeCast + | relationalOperator bitwiseOrExpression + )? + | 'super' relationalOperator bitwiseOrExpression + ; +relationalOperator + : '>=' + | '>' + | '<=' + | '<' + ; + +// 16.26 Bitwize Expression +bitwiseOrExpression + : bitwiseXorExpression ('|' bitwiseXorExpression)* + | 'super' ('|' bitwiseOrExpression)+ + ; +bitwiseXorExpression + : bitwiseAndExpression ('^' bitwiseAndExpression)* + | 'super' ('^' bitwiseAndExpression)+ + ; +bitwiseAndExpression + : shiftExpression ('&' shiftExpression)* + | 'super' ('&' shiftExpression)+ + ; +bitwiseOperator + : '&' + | '^' + | '|' + ; + +// 16.27 Shift +shiftExpression + : additiveExpression (shiftOperator additiveExpression)* + | 'super' (shiftOperator additiveExpression)+ + ; +shiftOperator + : '<<' + | '>>' + | '>>>' + ; + +// 16.28 Additive Expression +additiveExpression + : multiplicativeExpression (additiveOperator multiplicativeExpression)* + | 'super' (additiveOperator multiplicativeExpression)+ + ; +additiveOperator + : '+' + | '-' + ; +// 16.29 Multiplicative Expression +multiplicativeExpression + : unaryExpression (multiplicativeOperator unaryExpression)* + | 'super' (multiplicativeOperator unaryExpression)+ + ; +multiplicativeOperator + : '*' + | '/' + | '%' + | '~/' + ; + +// 16.30 Unary Expression +unaryExpression + : prefixOperator unaryExpression + | awaitExpression + | postfixExpression + | (minusOperator | tildeOperator) 'super' + | incrementOperator assignableExpression + ; +prefixOperator + : minusOperator + | negationOperator + | tildeOperator + ; +minusOperator: '-'; +negationOperator: '!'; +tildeOperator: '~'; + +// 16.31 Await Expressions +awaitExpression + : 'await' unaryExpression + ; + +// 16.32 Postfix Expressions +postfixExpression + : assignableExpression postfixOperator + | primary selector* + ; +postfixOperator + : incrementOperator + ; +selector + : assignableSelector + | argumentPart + ; + +incrementOperator + : '++' + | '--' + ; +// 16.33 Assignable Expressions +// NOTE +// primary (argumentPart* assignableSelector)+ -> primary (argumentPart* assignableSelector)? +assignableExpression + : primary (argumentPart* assignableSelector)? + | 'super' unconditionalAssignableSelector identifier + ; +unconditionalAssignableSelector + : '[' expression ']' + | '.' identifier + ; +assignableSelector + : unconditionalAssignableSelector + | '?.' identifier + ; + +identifier + : IDENTIFIER + ; +qualified + : identifier ('.' identifier)? + ; +// 16.35 Type Test +typeTest + : isOperator type + ; +isOperator + : 'is' '!'? + ; + +// 16.36 Type Cast +typeCast + : asOperator type + ; +asOperator + : 'as' + ; +// 17 Statements +statements + : statement* + ; +statement + : label* nonLabledStatment + ; +nonLabledStatment + : block + | localVariableDeclaration + | forStatement + | whileStatement + | doStatement + | switchStatement + | ifStatement + | rethrowStatment + | tryStatement + | breakStatement + | continueStatement + | returnStatement + | yieldStatement + | yieldEachStatement + | expressionStatement + | assertStatement + | localFunctionDeclaration + ; + +// 17.2 Expression Statements +expressionStatement + : expression? ';' + ; + +// 17.3 Local Variable Declaration +localVariableDeclaration + : initializedVariableDeclaration ';' + ; +// 17.4 Local Function Declaration +localFunctionDeclaration + : functionSignature functionBody + ; +// 17.5 If +ifStatement + : 'if' '(' expression ')' statement ('else' statement)? + ; + +// 17.6 For for +forStatement + : 'await'? 'for' '(' forLoopParts ')' statement + ; +forLoopParts + : forInitializerStatement expression? ';' expressionList? + | declaredIdentifier 'in' expression + | identifier 'in' expression + ; +forInitializerStatement + : localVariableDeclaration + | expression? ';' + ; + +// 17.7 While + +whileStatement + : 'while' '(' expression ')' statement + ; +// 17.8 Do +doStatement + : 'do' statement 'while' '(' expression ')' ';' + ; +// 17.9 Switch +switchStatement + : 'switch' '(' expression ')' '{' switchCase* defaultCase? '}' + ; +switchCase + : label* 'case' expression ':' statements + ; +defaultCase + : label* 'default' ':' statements + ; + +// 17.10 Rethrow +rethrowStatment + : 'rethrow' ';' + ; + +// 17.11 Try +tryStatement + : 'try' block (onPart+ finallyPart? | finallyPart) + ; +onPart + : catchPart block + | 'on' type catchPart? block + ; +catchPart + : 'catch' '(' identifier (',' identifier)? ')' + ; +finallyPart + : 'finally' block + ; + +// 17.12 Return + +returnStatement + : 'return' expression? ';' + ; + +// 17.13 Labels +label + : identifier ':' + ; + +// 17.13 Break +breakStatement + : 'break' identifier? ';' + ; + +// 17.13 Continue +continueStatement + : 'continue' identifier? ';' + ; + +// 17.16.1 Yield +yieldStatement + : 'yield' expression ';' + ; +// 17.16.1 Yield-Each +yieldEachStatement + : 'yield*' expression ';' + ; + +// 17.17 Assert +assertStatement + : assertion ';' + ; +assertion + : 'assert' '(' expression (',' expression )? ','? ')' + ; + +// 18 Libraries and Scripts +topLevelDefinition + : classDefinition + | enumType + | typeAlias + | 'external'? functionSignature ';' + | 'external'? getterSignature ';' + | 'external'? setterSignature ';' + | functionSignature functionBody + | returnType? 'get' identifier functionBody + | returnType? 'set' identifier formalParameterList functionBody + | ('final' | 'const') type? staticFinalDeclarationList ';' + | variableDeclaration ';' + ; +getOrSet + : 'get' + | 'set' + ; +libraryDefinition + : scriptTag? libraryName? importOrExport* partDirective* + topLevelDefinition* + ; +scriptTag + : '#!' (~NEWLINE)* NEWLINE + ; + +libraryName + : metadata 'library' dottedIdentifierList ';' + ; +importOrExport + : libraryimport + | libraryExport + ; +dottedIdentifierList + : identifier (',' identifier)* + ; + +libraryimport + : metadata importSpecification + ; + +importSpecification + : 'import' configurableUri ('as' identifier)? combinator* ';' +// | 'import' uri 'deferred' 'as' identifier combinator* ';' + ; + +combinator + : 'show' identifierList + | 'hide' identifierList + ; +identifierList + : identifier (',' identifier)* + ; + +// 18.2 Exports +libraryExport + : metadata 'export' configurableUri combinator* ';' + ; + +// 18.3 Parts +partDirective + : metadata 'part' uri ';' + ; +partHeader + : metadata 'part' 'of' identifier ('.' identifier)* ';' + ; +partDeclaration + : partHeader topLevelDefinition* EOF + ; + +// 18.5 URIs +uri + : stringLiteral + ; +configurableUri + : uri configurationUri* + ; +configurationUri + : 'if' '(' uriTest ')' uri + ; +uriTest + : dottedIdentifierList ('==' stringLiteral)? + ; + +// 19.1 Static Types +type + : typeName typeArguments? + ; +typeName + : qualified + | 'void' // SyntaxFix + ; +typeArguments + : '<' typeList '>' + ; +typeList + : type (',' type)* + ; + +// 19.3.1 Typedef +typeAlias + : metadata 'typedef' typeAliasBody + ; +typeAliasBody + : functionTypeAlias + ; +functionTypeAlias + : functionPrefix typeParameters? formalParameterList ';' + ; +functionPrefix + : returnType? identifier + ; + +// 20.2 Lexical Rules +// 20.1.1 Reserved Words +//assert, break, case, catch, class, const, continue, default, do, else, +//enum, extends, false, final, finally, for, if, in, is, new, null, rethrow, +//return, super, switch, this, throw, true, try, var, void, while, with. +fragment +IDENTIFIER_NO_DOLLAR + : IDENTIFIER_START_NO_DOLLAR + IDENTIFIER_PART_NO_DOLLAR* + ; +IDENTIFIER + : IDENTIFIER_START IDENTIFIER_PART* + ; + +//BUILT_IN_IDENTIFIER +// : 'abstract' +// | 'as' +// | 'covariant' +// | 'deferred' +// | 'dynamic' +// | 'export' +// | 'external' +// | 'factory' +// | 'Function' +// | 'get' +// | 'implements' +// | 'import' +// | 'interface' +// | 'library' +// | 'operator' +// | 'mixin' +// | 'part' +// | 'set' +// | 'static' +// | 'typedef' +// ; +fragment +IDENTIFIER_START + : IDENTIFIER_START_NO_DOLLAR + | '$' + ; +fragment +IDENTIFIER_START_NO_DOLLAR + : LETTER + | '_' + ; +fragment +IDENTIFIER_PART_NO_DOLLAR + : IDENTIFIER_START_NO_DOLLAR + | DIGIT + ; +fragment +IDENTIFIER_PART + : IDENTIFIER_START + | DIGIT + ; + +// 20.1.1 Reserved Words +fragment +LETTER + : [a-z] + | [A-Z] + ; +fragment +DIGIT + : [0-9] + ; +// 20.1.2 Comments +SINGLE_LINE_COMMENT +// : '//' ~(NEWLINE)* (NEWLINE)? // Origin Syntax + : '//' ~[\r\n]* -> skip + ; +MULTI_LINE_COMMENT +// : '/*' (MULTI_LINE_COMMENT | ~'*/')* '*/' // Origin Syntax + : '/*' .*? '*/' -> skip + ; + diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartLanguage.java b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartLanguage.java new file mode 100644 index 0000000000..1f4e8d4748 --- /dev/null +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartLanguage.java @@ -0,0 +1,18 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +/** + * Language implementation for Dart + */ +public class DartLanguage extends AbstractLanguage { + + /** + * Creates a new Dart Language instance. + */ + public DartLanguage() { + super("Dart", "dart", new DartTokenizer(), ".dart"); + } +} diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java new file mode 100644 index 0000000000..50aa47f84b --- /dev/null +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java @@ -0,0 +1,70 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import org.antlr.v4.runtime.CharStream; + +import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; +import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; +import net.sourceforge.pmd.lang.dart.antlr4.Dart2Lexer; + +/** + * The Dart Tokenizer + */ +public class DartTokenizer extends AntlrTokenizer { + + @Override + protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) { + CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); + return new AntlrTokenManager(new Dart2Lexer(charStream), sourceCode.getFileName()); + } + + @Override + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { + return new DartTokenFilter(tokenManager); + } + + /** + * The {@link DartTokenFilter} extends the {@link AntlrTokenFilter} to discard + * Dart-specific tokens. + *

+ * By default, it discards package and import statements, and + * enables annotation-based CPD suppression. + *

+ */ + private static class DartTokenFilter extends AntlrTokenFilter { + private boolean discardingPackageAndImport = false; + private boolean discardingNL = false; + + /* default */ DartTokenFilter(final AntlrTokenManager tokenManager) { + super(tokenManager); + } + + @Override + protected void analyzeToken(final AntlrToken currentToken) { + skipPackageAndImport(currentToken); + skipNewLines(currentToken); + } + + private void skipPackageAndImport(final AntlrToken currentToken) { + final int type = currentToken.getType(); + /*if (type == Dart.PACKAGE || type == Dart.IMPORT) { + discardingPackageAndImport = true; + } else if (discardingPackageAndImport && (type == Dart.SEMICOLON || type == Dart.NL)) { + discardingPackageAndImport = false; + }*/ + } + + private void skipNewLines(final AntlrToken currentToken) { + discardingNL = false;//currentToken.getType() == Dart.NL; + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return discardingPackageAndImport || discardingNL; + } + } +} diff --git a/pmd-dart/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language b/pmd-dart/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language new file mode 100644 index 0000000000..9c2eb85ed5 --- /dev/null +++ b/pmd-dart/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language @@ -0,0 +1 @@ +net.sourceforge.pmd.cpd.DartLanguage diff --git a/pmd-dist/pom.xml b/pmd-dist/pom.xml index 18282ae0b5..00cc4a1a0c 100644 --- a/pmd-dist/pom.xml +++ b/pmd-dist/pom.xml @@ -107,6 +107,11 @@ pmd-cs ${project.version} + + net.sourceforge.pmd + pmd-dart + ${project.version} + net.sourceforge.pmd pmd-fortran diff --git a/pom.xml b/pom.xml index 2efb4559fe..8ef49eaf73 100644 --- a/pom.xml +++ b/pom.xml @@ -1130,6 +1130,7 @@ Additionally it includes CPD, the copy-paste-detector. CPD finds duplicated code pmd-core pmd-cpp pmd-cs + pmd-dart pmd-dist pmd-fortran pmd-go From 745ebc461c79be33e9eacdcba20a97768ad5fec8 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Fri, 5 Apr 2019 17:04:49 +0200 Subject: [PATCH 10/23] Added test cases for Dart support for CPD. --- .../pmd/cpd/DartTokenizerTest.java | 56 +++++++++++++++++++ .../net/sourceforge/pmd/cpd/comment.dart | 26 +++++++++ .../net/sourceforge/pmd/cpd/imports.dart | 15 +++++ .../net/sourceforge/pmd/cpd/increment.dart | 24 ++++++++ 4 files changed, 121 insertions(+) create mode 100644 pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java create mode 100644 pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/comment.dart create mode 100644 pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/imports.dart create mode 100644 pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/increment.dart diff --git a/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java b/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java new file mode 100644 index 0000000000..02cf0a92f0 --- /dev/null +++ b/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java @@ -0,0 +1,56 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import net.sourceforge.pmd.testframework.AbstractTokenizerTest; + +@RunWith(Parameterized.class) +public class DartTokenizerTest extends AbstractTokenizerTest { + + private final String filename; + private final int nExpectedTokens; + + public DartTokenizerTest(String filename, int nExpectedTokens) { + this.filename = filename; + this.nExpectedTokens = nExpectedTokens; + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + new Object[] { "comment.dart" , 5 }, + new Object[] { "increment.dart" , 185 }, + new Object[] { "imports.dart" , 1 } + ); + } + + @Before + @Override + public void buildTokenizer() throws IOException { + this.tokenizer = new DartTokenizer(); + this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), this.filename)); + } + + @Override + public String getSampleCode() throws IOException { + return IOUtils.toString(DartTokenizer.class.getResourceAsStream(this.filename)); + } + + @Test + public void tokenizeTest() throws IOException { + this.expectedTokenCount = nExpectedTokens; + super.tokenizeTest(); + } +} diff --git a/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/comment.dart b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/comment.dart new file mode 100644 index 0000000000..d5081bc3e5 --- /dev/null +++ b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/comment.dart @@ -0,0 +1,26 @@ +library mylibrary; + +var x = 0; + +/* +void increment1() { x += 1; } +void increment2() { x += 1; } +void increment3() { x += 1; } +void increment4() { x += 1; } +void increment5() { x += 1; } +void increment6() { x += 1; } +void increment7() { x += 1; } +void increment8() { x += 1; } +void increment9() { x += 1; } +void increment10() { x += 1; } +void increment11() { x += 1; } +void increment12() { x += 1; } +void increment13() { x += 1; } +void increment14() { x += 1; } +void increment15() { x += 1; } +void increment16() { x += 1; } +void increment17() { x += 1; } +void increment18() { x += 1; } +void increment19() { x += 1; } +void increment20() { x += 1; } +*/ \ No newline at end of file diff --git a/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/imports.dart b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/imports.dart new file mode 100644 index 0000000000..cc8aefbf2f --- /dev/null +++ b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/imports.dart @@ -0,0 +1,15 @@ +import 'package:flutter/material.dart'; +import 'package:english_words/english_words.dart' as EnglishWords; +import 'dart:io' deferred as io; +import 'dart:async' show AsyncError hide Completer; +import 'dart:async' hide Completer show AsyncError; +import 'dart:async' show AsyncError; +import 'dart:async' hide Completer; +import 'dart:async' show AsyncError, Completer hide ControllerCallback, ControllerCancelCallback; +import 'dart:async' show AsyncError, Completer; +import 'dart:async' hide ControllerCallback, ControllerCancelCallback; +import "package:boolean_selector/boolean_selector.dart"; +import '''package:charcode/ascii.dart''' as ascii; +import """package:collection/algorithms.dart""" as algo; +import 'dart:collection' show HashMap; +import 'dart:math' hide ln10 show ln2, cos, sin; diff --git a/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/increment.dart b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/increment.dart new file mode 100644 index 0000000000..d12e6d400f --- /dev/null +++ b/pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/increment.dart @@ -0,0 +1,24 @@ +library mylibrary; + +var x = 0; + +void increment1() { x += 1; } +void increment2() { x += 1; } +void increment3() { x += 1; } +void increment4() { x += 1; } +void increment5() { x += 1; } +void increment6() { x += 1; } +void increment7() { x += 1; } +void increment8() { x += 1; } +void increment9() { x += 1; } +void increment10() { x += 1; } +void increment11() { x += 1; } +void increment12() { x += 1; } +void increment13() { x += 1; } +void increment14() { x += 1; } +void increment15() { x += 1; } +void increment16() { x += 1; } +void increment17() { x += 1; } +void increment18() { x += 1; } +void increment19() { x += 1; } +void increment20() { x += 1; } \ No newline at end of file From 1a16d1a52da0598bbb7d2bd2b9f895eac0ea7920 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Fri, 5 Apr 2019 17:05:26 +0200 Subject: [PATCH 11/23] Adjusted Dart tokenizer to skip over uninteresting changes. --- .../sourceforge/pmd/lang/dart/antlr4/Dart2.g4 | 176 ++++++++++-------- .../sourceforge/pmd/cpd/DartTokenizer.java | 26 ++- 2 files changed, 116 insertions(+), 86 deletions(-) diff --git a/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 b/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 index 78d3083d2c..1c3dffa8c6 100644 --- a/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 +++ b/pmd-dart/src/main/antlr4/net/sourceforge/pmd/lang/dart/antlr4/Dart2.g4 @@ -35,6 +35,8 @@ WHITESPACE : [ \t\r\n\u000C]+ -> skip ; +SEMICOLON: ';' ; + // 8 Variables variableDeclaration : declaredIdentifier (',' identifier)* @@ -79,7 +81,7 @@ returnType ; functionBody - : 'async'? '=>' expression ';' + : 'async'? '=>' expression SEMICOLON | ('async' | 'async*' | 'sync*')? block ; block @@ -114,11 +116,11 @@ normalFormalParameter | simpleFormalParameter ; functionFormalParameter - : metadata 'covariant'? returnType? identifier formalParameterPart + : metadata COVARIANT? returnType? identifier formalParameterPart ; simpleFormalParameter : declaredIdentifier - | metadata 'covariant'? identifier + | metadata COVARIANT? identifier ; fieldFormalParameter : metadata finalConstVarOrType? 'this' '.' identifier formalParameterPart? @@ -135,24 +137,24 @@ defaultNamedParameter // 10 Classes classDefinition - : metadata 'abstract'? 'class' identifier typeParameters? + : metadata ABSTRACT? 'class' identifier typeParameters? superclass? mixins? interfaces? '{' (metadata classMemberDefinition)* '}' - | metadata 'abstract'? 'class' mixinApplicationClass + | metadata ABSTRACT? 'class' mixinApplicationClass ; mixins : 'with' typeList ; classMemberDefinition - : declaration ';' + : declaration SEMICOLON | methodSignature functionBody ; methodSignature : constructorSignature initializers? | factoryConstructorSignature - | 'static'? functionSignature - | 'static'? getterSignature - | 'static'? setterSignature + | STATIC? functionSignature + | STATIC? getterSignature + | STATIC? setterSignature | operatorSignature ; @@ -160,15 +162,15 @@ methodSignature declaration : constantConstructorSignature (redirection | initializers)? | constructorSignature (redirection | initializers)? - | 'external' constantConstructorSignature - | 'external' constructorSignature - | ('external' 'static'?)? getterSignature - | ('external' 'static'?)? setterSignature - | 'external'? operatorSignature - | ('external' 'static'?)? functionSignature - | 'static' ('final' | 'const') type? staticFinalDeclarationList + | EXTERNAL constantConstructorSignature + | EXTERNAL constructorSignature + | (EXTERNAL STATIC?)? getterSignature + | (EXTERNAL STATIC?)? setterSignature + | EXTERNAL? operatorSignature + | (EXTERNAL STATIC?)? functionSignature + | STATIC ('final' | 'const') type? staticFinalDeclarationList | 'final' type? initializedIdentifierList - | ('static' | 'covariant')? ('var' | type) initializedIdentifierList + | (STATIC | COVARIANT)? ('var' | type) initializedIdentifierList ; staticFinalDeclarationList @@ -180,7 +182,7 @@ staticFinalDeclaration // 10.1.1 Operators operatorSignature - : returnType? 'operator' operator formalParameterList + : returnType? OPERATOR operator formalParameterList ; operator : '~' | binaryOperator | '[]' | '[]=' @@ -196,11 +198,11 @@ binaryOperator ; // 10.2 Getters getterSignature - : returnType? 'get' identifier + : returnType? GET identifier ; // 10.2 Setters setterSignature - : returnType? 'set' identifier formalParameterList + : returnType? SET identifier formalParameterList ; // 10.6 Constructors @@ -226,10 +228,10 @@ fieldInitializer // 10.6.2 Factories factoryConstructorSignature - : 'factory' identifier ('.' identifier)? formalParameterList + : FACTORY identifier ('.' identifier)? formalParameterList ; redirectingFactoryConstructorSignature - : 'const'? 'factory' identifier ('.' identifier)? formalParameterList '=' + : 'const'? FACTORY identifier ('.' identifier)? formalParameterList '=' type ('.' identifier)? ; // 10.6.3 Constant Constructors @@ -239,11 +241,11 @@ constantConstructorSignature: 'const' qualified formalParameterList; superclass: 'extends' type; // 10.10 SUperinterfaces -interfaces: 'implements' typeList; +interfaces: IMPLEMENTS typeList; // 12.1 Mixin Application mixinApplicationClass - : identifier typeParameters? '=' mixinApplication ';'; + : identifier typeParameters? '=' mixinApplication SEMICOLON; mixinApplication : type mixins interfaces? ; @@ -664,7 +666,7 @@ typeCast : asOperator type ; asOperator - : 'as' + : AS ; // 17 Statements statements @@ -695,12 +697,12 @@ nonLabledStatment // 17.2 Expression Statements expressionStatement - : expression? ';' + : expression? SEMICOLON ; // 17.3 Local Variable Declaration localVariableDeclaration - : initializedVariableDeclaration ';' + : initializedVariableDeclaration SEMICOLON ; // 17.4 Local Function Declaration localFunctionDeclaration @@ -716,13 +718,13 @@ forStatement : 'await'? 'for' '(' forLoopParts ')' statement ; forLoopParts - : forInitializerStatement expression? ';' expressionList? + : forInitializerStatement expression? SEMICOLON expressionList? | declaredIdentifier 'in' expression | identifier 'in' expression ; forInitializerStatement : localVariableDeclaration - | expression? ';' + | expression? SEMICOLON ; // 17.7 While @@ -732,7 +734,7 @@ whileStatement ; // 17.8 Do doStatement - : 'do' statement 'while' '(' expression ')' ';' + : 'do' statement 'while' '(' expression ')' SEMICOLON ; // 17.9 Switch switchStatement @@ -747,7 +749,7 @@ defaultCase // 17.10 Rethrow rethrowStatment - : 'rethrow' ';' + : 'rethrow' SEMICOLON ; // 17.11 Try @@ -768,7 +770,7 @@ finallyPart // 17.12 Return returnStatement - : 'return' expression? ';' + : 'return' expression? SEMICOLON ; // 17.13 Labels @@ -778,26 +780,26 @@ label // 17.13 Break breakStatement - : 'break' identifier? ';' + : 'break' identifier? SEMICOLON ; // 17.13 Continue continueStatement - : 'continue' identifier? ';' + : 'continue' identifier? SEMICOLON ; // 17.16.1 Yield yieldStatement - : 'yield' expression ';' + : 'yield' expression SEMICOLON ; // 17.16.1 Yield-Each yieldEachStatement - : 'yield*' expression ';' + : 'yield*' expression SEMICOLON ; // 17.17 Assert assertStatement - : assertion ';' + : assertion SEMICOLON ; assertion : 'assert' '(' expression (',' expression )? ','? ')' @@ -808,18 +810,18 @@ topLevelDefinition : classDefinition | enumType | typeAlias - | 'external'? functionSignature ';' - | 'external'? getterSignature ';' - | 'external'? setterSignature ';' + | EXTERNAL? functionSignature SEMICOLON + | EXTERNAL? getterSignature SEMICOLON + | EXTERNAL? setterSignature SEMICOLON | functionSignature functionBody - | returnType? 'get' identifier functionBody - | returnType? 'set' identifier formalParameterList functionBody - | ('final' | 'const') type? staticFinalDeclarationList ';' - | variableDeclaration ';' + | returnType? GET identifier functionBody + | returnType? SET identifier formalParameterList functionBody + | ('final' | 'const') type? staticFinalDeclarationList SEMICOLON + | variableDeclaration SEMICOLON ; getOrSet - : 'get' - | 'set' + : GET + | SET ; libraryDefinition : scriptTag? libraryName? importOrExport* partDirective* @@ -830,7 +832,7 @@ scriptTag ; libraryName - : metadata 'library' dottedIdentifierList ';' + : metadata LIBRARY dottedIdentifierList SEMICOLON ; importOrExport : libraryimport @@ -845,8 +847,8 @@ libraryimport ; importSpecification - : 'import' configurableUri ('as' identifier)? combinator* ';' -// | 'import' uri 'deferred' 'as' identifier combinator* ';' + : IMPORT configurableUri (AS identifier)? combinator* SEMICOLON +// | IMPORT uri DEFERRED AS identifier combinator* SEMICOLON ; combinator @@ -859,15 +861,15 @@ identifierList // 18.2 Exports libraryExport - : metadata 'export' configurableUri combinator* ';' + : metadata EXPORT configurableUri combinator* SEMICOLON ; // 18.3 Parts partDirective - : metadata 'part' uri ';' + : metadata PART uri SEMICOLON ; partHeader - : metadata 'part' 'of' identifier ('.' identifier)* ';' + : metadata PART 'of' identifier ('.' identifier)* SEMICOLON ; partDeclaration : partHeader topLevelDefinition* EOF @@ -904,13 +906,13 @@ typeList // 19.3.1 Typedef typeAlias - : metadata 'typedef' typeAliasBody + : metadata TYPEDEF typeAliasBody ; typeAliasBody : functionTypeAlias ; functionTypeAlias - : functionPrefix typeParameters? formalParameterList ';' + : functionPrefix typeParameters? formalParameterList SEMICOLON ; functionPrefix : returnType? identifier @@ -921,6 +923,50 @@ functionPrefix //assert, break, case, catch, class, const, continue, default, do, else, //enum, extends, false, final, finally, for, if, in, is, new, null, rethrow, //return, super, switch, this, throw, true, try, var, void, while, with. +ABSTRACT: 'abstract' ; +AS: 'as' ; +COVARIANT: 'covariant' ; +DEFERRED: 'deferred' ; +DYNAMIC: 'dynamic' ; +EXPORT: 'export' ; +EXTERNAL: 'external' ; +FACTORY: 'factory' ; +FUNCTION: 'Function' ; +GET: 'get' ; +IMPLEMENTS: 'implements' ; +IMPORT: 'import' ; +INTERFACE: 'interface' ; +LIBRARY: 'library' ; +OPERATOR: 'operator' ; +MIXIN: 'mixin' ; +PART: 'part' ; +SET: 'set' ; +STATIC: 'static' ; +TYPEDEF: 'typedef' ; + +//BUILT_IN_IDENTIFIER +// : ABSTRACT +// | AS +// | COVARIANT +// | DEFERRED +// | DYNAMIC +// | EXPORT +// | EXTERNAL +// | FACTORY +// | FUNCTION +// | GET +// | IMPLEMENTS +// | IMPORT +// | INTERFACE +// | LIBRARY +// | OPERATOR +// | MIXIN +// | PART +// | SET +// | STATIC +// | TYPEDEF +// ; + fragment IDENTIFIER_NO_DOLLAR : IDENTIFIER_START_NO_DOLLAR @@ -930,28 +976,6 @@ IDENTIFIER : IDENTIFIER_START IDENTIFIER_PART* ; -//BUILT_IN_IDENTIFIER -// : 'abstract' -// | 'as' -// | 'covariant' -// | 'deferred' -// | 'dynamic' -// | 'export' -// | 'external' -// | 'factory' -// | 'Function' -// | 'get' -// | 'implements' -// | 'import' -// | 'interface' -// | 'library' -// | 'operator' -// | 'mixin' -// | 'part' -// | 'set' -// | 'static' -// | 'typedef' -// ; fragment IDENTIFIER_START : IDENTIFIER_START_NO_DOLLAR diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java index 50aa47f84b..a29a84b05b 100644 --- a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java @@ -36,8 +36,9 @@ public class DartTokenizer extends AntlrTokenizer { *

*/ private static class DartTokenFilter extends AntlrTokenFilter { - private boolean discardingPackageAndImport = false; + private boolean discardingLibraryAndImport = false; private boolean discardingNL = false; + private boolean discardingSemicolon = false; /* default */ DartTokenFilter(final AntlrTokenManager tokenManager) { super(tokenManager); @@ -45,26 +46,31 @@ public class DartTokenizer extends AntlrTokenizer { @Override protected void analyzeToken(final AntlrToken currentToken) { - skipPackageAndImport(currentToken); + skipLibraryAndImport(currentToken); skipNewLines(currentToken); + skipSemicolons(currentToken); } - private void skipPackageAndImport(final AntlrToken currentToken) { + private void skipLibraryAndImport(final AntlrToken currentToken) { final int type = currentToken.getType(); - /*if (type == Dart.PACKAGE || type == Dart.IMPORT) { - discardingPackageAndImport = true; - } else if (discardingPackageAndImport && (type == Dart.SEMICOLON || type == Dart.NL)) { - discardingPackageAndImport = false; - }*/ + if (type == Dart2Lexer.LIBRARY || type == Dart2Lexer.IMPORT) { + discardingLibraryAndImport = true; + } else if (discardingLibraryAndImport && (type == Dart2Lexer.SEMICOLON || type == Dart2Lexer.NEWLINE)) { + discardingLibraryAndImport = false; + } } private void skipNewLines(final AntlrToken currentToken) { - discardingNL = false;//currentToken.getType() == Dart.NL; + discardingNL = currentToken.getType() == Dart2Lexer.NEWLINE; + } + + private void skipSemicolons(final AntlrToken currentToken) { + discardingSemicolon = currentToken.getType() == Dart2Lexer.SEMICOLON; } @Override protected boolean isLanguageSpecificDiscarding() { - return discardingPackageAndImport || discardingNL; + return discardingLibraryAndImport || discardingNL || discardingSemicolon; } } } From 1e31e792f9abf1c318a677c38c510fb00e423571 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Fri, 5 Apr 2019 17:22:04 +0200 Subject: [PATCH 12/23] Adjusted pmd-dist unit test to include Dart in the list of supported languages. --- .../test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-dist/src/test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java b/pmd-dist/src/test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java index b37093098f..5da0b5f5a1 100644 --- a/pmd-dist/src/test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java +++ b/pmd-dist/src/test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java @@ -112,7 +112,7 @@ public class BinaryDistributionIT { result = CpdExecutor.runCpd(tempDir, "-h"); - result.assertExecutionResult(1, "Supported languages: [apex, cpp, cs, ecmascript, fortran, go, groovy, java, jsp, kotlin, matlab, objectivec, perl, php, plsql, python, ruby, scala, swift, vf]"); + result.assertExecutionResult(1, "Supported languages: [apex, cpp, cs, dart, ecmascript, fortran, go, groovy, java, jsp, kotlin, matlab, objectivec, perl, php, plsql, python, ruby, scala, swift, vf]"); result = CpdExecutor.runCpd(tempDir, "--minimum-tokens", "10", "--format", "text", "--files", srcDir); result.assertExecutionResult(4, "Found a 10 line (55 tokens) duplication in the following files:"); From 7b78ac75d0e6a9bb4c0fa59a553dc9c63f937638 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Fri, 5 Apr 2019 17:22:31 +0200 Subject: [PATCH 13/23] Minor formatting changes. --- .../java/net/sourceforge/pmd/cpd/DartTokenizerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java b/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java index 02cf0a92f0..397613f1fd 100644 --- a/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java +++ b/pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java @@ -30,9 +30,9 @@ public class DartTokenizerTest extends AbstractTokenizerTest { @Parameterized.Parameters public static Collection data() { return Arrays.asList( - new Object[] { "comment.dart" , 5 }, - new Object[] { "increment.dart" , 185 }, - new Object[] { "imports.dart" , 1 } + new Object[] { "comment.dart", 5 }, + new Object[] { "increment.dart", 185 }, + new Object[] { "imports.dart", 1 } ); } From b27d1de001d22b67d038aa30d02d767c5b5be255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 12:37:25 -0300 Subject: [PATCH 14/23] Add kotlin to supported languages --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0abdf0b0dd..3b007fbc9b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ unnecessary object creation, and so forth. It supports Java, JavaScript, Salesfo XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in -Java, C, C++, C#, Groovy, PHP, Ruby, Fortran, JavaScript, PLSQL, Apache Velocity, Scala, Objective C, +Java, C, C++, C#, Groovy, Kotlin, PHP, Ruby, Fortran, JavaScript, PLSQL, Apache Velocity, Scala, Objective C, Salesforce.com Apex, Perl, Swift, Matlab, Python. ## Source and Documentation From 9d550a1224e8fcb1e2beae82b466c9f6e4104c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 12:40:20 -0300 Subject: [PATCH 15/23] Add go to supported languages, sort alphabetically --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3b007fbc9b..2f93da6e18 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ unnecessary object creation, and so forth. It supports Java, JavaScript, Salesfo XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in -Java, C, C++, C#, Groovy, Kotlin, PHP, Ruby, Fortran, JavaScript, PLSQL, Apache Velocity, Scala, Objective C, -Salesforce.com Apex, Perl, Swift, Matlab, Python. +Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, Kotlin, Matlab, +Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift. ## Source and Documentation From 62c44253216c6c9dacaff6acb9b424a02cf9064e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 12:41:02 -0300 Subject: [PATCH 16/23] Add visualforce to supported languages --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f93da6e18..2473289361 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, Kotlin, Matlab, -Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift. +Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift and Visualforce. ## Source and Documentation From 3884d4cfbc2b306e718dfeeb4505a2287f919532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 12:42:04 -0300 Subject: [PATCH 17/23] Add JSP to CPD supported languages --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2473289361..c880e7ce3c 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ unnecessary object creation, and so forth. It supports Java, JavaScript, Salesfo XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in -Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, Kotlin, Matlab, +Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift and Visualforce. ## Source and Documentation From 4316ecaa5cde6cc593e7a102fcedb19c6f20a9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 12:43:52 -0300 Subject: [PATCH 18/23] Add XML/XSL to CPD supported languages --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c880e7ce3c..c80a603ee0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, -Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift and Visualforce. +Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift, Visualforce, XML and XSL. ## Source and Documentation From ee227019aaa5972afe2b3b24ca2514dba821f55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 16:17:25 -0300 Subject: [PATCH 19/23] Update changelog, refs #1761 --- docs/pages/release_notes.md | 11 +++++++++++ .../java/net/sourceforge/pmd/cpd/DartTokenizer.java | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 8b53655c31..f213d9ac41 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,16 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### Dart support + +Thanks to the contribution from [Maikel Steneker](https://github.com/maikelsteneker), and built on top of the ongoing efforts to fully support Antlr-based languages, +PMD now has CPD support for [Dart](https://www.dartlang.org/). + +Being based on a proper Antlr grammar, CPD can: +* ignore comments +* ignore imports / libraries +* honor [comment-based suppressions](pmd_userdocs_cpd.html#suppression) + ### Fixed Issues * java @@ -27,6 +37,7 @@ This is a {{ site.pmd.release_type }} release. * [#1745](https://github.com/pmd/pmd/pull/1745): \[doc] Fixed some errors in docs - [0xflotus](https://github.com/0xflotus) * [#1746](https://github.com/pmd/pmd/pull/1746): \[java] Update rule to prevent UnusedImport when using JavaDoc with array type - [itaigilo](https://github.com/itaigilo) +* [#1761](https://github.com/pmd/pmd/pull/1761): \[dart] \[cpd] Added CPD support for Dart - [Maikel Steneker](https://github.com/maikelsteneker) {% endtocmaker %} diff --git a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java index a29a84b05b..d8a53823d6 100644 --- a/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java +++ b/pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java @@ -32,7 +32,7 @@ public class DartTokenizer extends AntlrTokenizer { * Dart-specific tokens. *

* By default, it discards package and import statements, and - * enables annotation-based CPD suppression. + * enables comment-based CPD suppression. *

*/ private static class DartTokenFilter extends AntlrTokenFilter { From 8f7ce8bfe7410f8e9c3e6e9790f7029e91b9057f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 16:21:19 -0300 Subject: [PATCH 20/23] =?UTF-8?q?VM=20is=20not=20supported=20in=20CPD,=20b?= =?UTF-8?q?ut=20it's=20on=20PMD=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c80a603ee0..a9d9f45e37 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ unnecessary object creation, and so forth. It supports Java, JavaScript, Salesfo XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in -Apache Velocity, C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, +C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift, Visualforce, XML and XSL. ## Source and Documentation From 0db8697994ec77cacb53ecf09cd75469d7ea067f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 16:22:38 -0300 Subject: [PATCH 21/23] Remove XSL / XML from CPD support --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9d9f45e37..d182a092e0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, -Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift, Visualforce, XML and XSL. +Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift and Visualforce. ## Source and Documentation From 0328d5464ffa90e231aa37985c28f448a70ce33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 16:23:13 -0300 Subject: [PATCH 22/23] Revamp CPD userdocs --- docs/pages/pmd/userdocs/cpd.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/pages/pmd/userdocs/cpd.md b/docs/pages/pmd/userdocs/cpd.md index 65639bd1c0..492776d5df 100644 --- a/docs/pages/pmd/userdocs/cpd.md +++ b/docs/pages/pmd/userdocs/cpd.md @@ -11,7 +11,7 @@ author: Tom Copeland Duplicate code can be hard to find, especially in a large project. But PMD's **Copy/Paste Detector (CPD)** can find it for you! -CPD works with Java, JSP, C, C++, C#, Fortran and PHP code and [some more languages](#supported-languages). +CPD works with Java, JSP, C/C++, C#, Go, Kotlin, Ruby, Swift and [many more languages](#supported-languages). It can be used via [command-line](#cli-usage), or via an [Ant task](#ant-task). It can also be run with Maven by using the `cpd-check` goal on the [Maven PMD Plugin](pmd_userdocs_tools_maven.html). @@ -210,12 +210,14 @@ This behavior has been introduced to ease CPD integration into scripts or hooks, * Apex * C# * C/C++ +* Dart * EcmaScript (JavaScript) * Fortran * Go * Groovy * Java * Jsp +* Kotlin * Matlab * Objective-C * Perl @@ -363,7 +365,7 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package: ## Suppression -Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**, +Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Dart**, **Go**, **Javascript**, **Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`. ```java From 595c440465201e0706cfc09f09b2599472a0015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?= Date: Wed, 10 Apr 2019 16:25:16 -0300 Subject: [PATCH 23/23] Add Dart to CPD supported languages --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d182a092e0..b2ff396ea5 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ unnecessary object creation, and so forth. It supports Java, JavaScript, Salesfo XML, XSL. Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in -C/C++, C#, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, +C/C++, C#, Dart, Fortran, Go, Groovy, Java, JavaScript, JSP, Kotlin, Matlab, Objective-C, Perl, PHP, PLSQL, Python, Ruby, Salesforce.com Apex, Scala, Swift and Visualforce. ## Source and Documentation