Merge branch 'pr-1761'

docs/pages/pmd/userdocs/cpd.md

@@ -11,7 +11,7 @@ author: Tom Copeland <tom@infoether.com>
 Duplicate code can be hard to find, especially in a large project.
 But PMD's **Copy/Paste Detector (CPD)** can find it for you!
 
-CPD works with Java, JSP, C, C++, C#, Fortran and PHP code and [some more languages](#supported-languages).
+CPD works with Java, JSP, C/C++, C#, Go, Kotlin, Ruby, Swift and [many more languages](#supported-languages).
 It can be used via [command-line](#cli-usage), or via an [Ant task](#ant-task).
 It can also be run with Maven by using the `cpd-check` goal on the [Maven PMD Plugin](pmd_userdocs_tools_maven.html).
 
@@ -210,12 +210,14 @@ This behavior has been introduced to ease CPD integration into scripts or hooks,
 * Apex
 * C#
 * C/C++
+* Dart
 * EcmaScript (JavaScript)
 * Fortran
 * Go
 * Groovy
 * Java
 * Jsp
+* Kotlin
 * Matlab
 * Objective-C
 * Perl
@@ -363,7 +365,7 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package:
 
 ## Suppression
 
-Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Go**, **Javascript**,
+Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Dart**, **Go**, **Javascript**,
 **Kotlin**, **Matlab**, **Objective-C**, **PL/SQL**, **Python** and **Swift** by including the keywords `CPD-OFF` and `CPD-ON`.
 
 ```java

docs/pages/release_notes.md

@@ -14,6 +14,16 @@ This is a {{ site.pmd.release_type }} release.
 
 ### New and noteworthy
 
+#### Dart support
+
+Thanks to the contribution from [Maikel Steneker](https://github.com/maikelsteneker), and built on top of the ongoing efforts to fully support Antlr-based languages,
+PMD now has CPD support for [Dart](https://www.dartlang.org/).
+
+Being based on a proper Antlr grammar, CPD can:
+*   ignore comments
+*   ignore imports / libraries
+*   honor [comment-based suppressions](pmd_userdocs_cpd.html#suppression)
+
 ### Fixed Issues
 
 *   go
@@ -32,6 +42,7 @@ This is a {{ site.pmd.release_type }} release.
 *   [#1745](https://github.com/pmd/pmd/pull/1745): \[doc] Fixed some errors in docs - [0xflotus](https://github.com/0xflotus)
 *   [#1746](https://github.com/pmd/pmd/pull/1746): \[java] Update rule to prevent UnusedImport when using JavaDoc with array type - [itaigilo](https://github.com/itaigilo)
 *   [#1752](https://github.com/pmd/pmd/pull/1752): \[java] UseObjectForClearerAPI Only For Public - [Björn Kautler](https://github.com/Vampire)
+*   [#1761](https://github.com/pmd/pmd/pull/1761): \[dart] \[cpd] Added CPD support for Dart - [Maikel Steneker](https://github.com/maikelsteneker)
 
 {% endtocmaker %}
 

pmd-dart/pom.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>pmd-dart</artifactId>
+    <name>PMD Dart</name>
+
+    <parent>
+        <groupId>net.sourceforge.pmd</groupId>
+        <artifactId>pmd</artifactId>
+        <version>6.14.0-SNAPSHOT</version>
+    </parent>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-maven-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+                <configuration>
+                    <useDefaultDelimiters>false</useDefaultDelimiters>
+                    <delimiters>
+                        <delimiter>${*}</delimiter>
+                    </delimiters>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr4-runtime</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>net.sourceforge.pmd</groupId>
+            <artifactId>pmd-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.sourceforge.pmd</groupId>
+            <artifactId>pmd-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartLanguage.java

@@ -0,0 +1,18 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+/**
+ * Language implementation for Dart
+ */
+public class DartLanguage extends AbstractLanguage {
+
+    /**
+     * Creates a new Dart Language instance.
+     */
+    public DartLanguage() {
+        super("Dart", "dart", new DartTokenizer(), ".dart");
+    }
+}

pmd-dart/src/main/java/net/sourceforge/pmd/cpd/DartTokenizer.java

@@ -0,0 +1,76 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import org.antlr.v4.runtime.CharStream;
+
+import net.sourceforge.pmd.cpd.token.AntlrToken;
+import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
+import net.sourceforge.pmd.lang.antlr.AntlrTokenManager;
+import net.sourceforge.pmd.lang.dart.antlr4.Dart2Lexer;
+
+/**
+ * The Dart Tokenizer
+ */
+public class DartTokenizer extends AntlrTokenizer {
+
+    @Override
+    protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) {
+        CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode);
+        return new AntlrTokenManager(new Dart2Lexer(charStream), sourceCode.getFileName());
+    }
+
+    @Override
+    protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
+        return new DartTokenFilter(tokenManager);
+    }
+
+    /**
+     * The {@link DartTokenFilter} extends the {@link AntlrTokenFilter} to discard
+     * Dart-specific tokens.
+     * <p>
+     * By default, it discards package and import statements, and
+     * enables comment-based CPD suppression.
+     * </p>
+     */
+    private static class DartTokenFilter extends AntlrTokenFilter {
+        private boolean discardingLibraryAndImport = false;
+        private boolean discardingNL = false;
+        private boolean discardingSemicolon = false;
+
+        /* default */ DartTokenFilter(final AntlrTokenManager tokenManager) {
+            super(tokenManager);
+        }
+
+        @Override
+        protected void analyzeToken(final AntlrToken currentToken) {
+            skipLibraryAndImport(currentToken);
+            skipNewLines(currentToken);
+            skipSemicolons(currentToken);
+        }
+
+        private void skipLibraryAndImport(final AntlrToken currentToken) {
+            final int type = currentToken.getType();
+            if (type == Dart2Lexer.LIBRARY || type == Dart2Lexer.IMPORT) {
+                discardingLibraryAndImport = true;
+            } else if (discardingLibraryAndImport && (type == Dart2Lexer.SEMICOLON || type == Dart2Lexer.NEWLINE)) {
+                discardingLibraryAndImport = false;
+            }
+        }
+
+        private void skipNewLines(final AntlrToken currentToken) {
+            discardingNL = currentToken.getType() == Dart2Lexer.NEWLINE;
+        }
+
+        private void skipSemicolons(final AntlrToken currentToken) {
+            discardingSemicolon = currentToken.getType() == Dart2Lexer.SEMICOLON;
+        }
+
+        @Override
+        protected boolean isLanguageSpecificDiscarding() {
+            return discardingLibraryAndImport || discardingNL || discardingSemicolon;
+        }
+    }
+}

pmd-dart/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language

@@ -0,0 +1 @@
+net.sourceforge.pmd.cpd.DartLanguage

pmd-dart/src/test/java/net/sourceforge/pmd/cpd/DartTokenizerTest.java

@@ -0,0 +1,56 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
+
+@RunWith(Parameterized.class)
+public class DartTokenizerTest extends AbstractTokenizerTest {
+
+    private final String filename;
+    private final int nExpectedTokens;
+
+    public DartTokenizerTest(String filename, int nExpectedTokens) {
+        this.filename = filename;
+        this.nExpectedTokens = nExpectedTokens;
+    }
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> data() {
+        return Arrays.asList(
+                new Object[] { "comment.dart", 5 },
+                new Object[] { "increment.dart", 185 },
+                new Object[] { "imports.dart", 1 }
+        );
+    }
+
+    @Before
+    @Override
+    public void buildTokenizer() throws IOException {
+        this.tokenizer = new DartTokenizer();
+        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), this.filename));
+    }
+
+    @Override
+    public String getSampleCode() throws IOException {
+        return IOUtils.toString(DartTokenizer.class.getResourceAsStream(this.filename));
+    }
+
+    @Test
+    public void tokenizeTest() throws IOException {
+        this.expectedTokenCount = nExpectedTokens;
+        super.tokenizeTest();
+    }
+}

pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/comment.dart

@@ -0,0 +1,26 @@
+library mylibrary;
+
+var x = 0;
+
+/*
+void increment1() { x += 1; }
+void increment2() { x += 1; }
+void increment3() { x += 1; }
+void increment4() { x += 1; }
+void increment5() { x += 1; }
+void increment6() { x += 1; }
+void increment7() { x += 1; }
+void increment8() { x += 1; }
+void increment9() { x += 1; }
+void increment10() { x += 1; }
+void increment11() { x += 1; }
+void increment12() { x += 1; }
+void increment13() { x += 1; }
+void increment14() { x += 1; }
+void increment15() { x += 1; }
+void increment16() { x += 1; }
+void increment17() { x += 1; }
+void increment18() { x += 1; }
+void increment19() { x += 1; }
+void increment20() { x += 1; }
+*/

pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/imports.dart

@@ -0,0 +1,15 @@
+import 'package:flutter/material.dart';
+import 'package:english_words/english_words.dart' as EnglishWords;
+import 'dart:io' deferred as io;
+import 'dart:async' show AsyncError hide Completer;
+import 'dart:async' hide Completer show AsyncError;
+import 'dart:async' show AsyncError;
+import 'dart:async' hide Completer;
+import 'dart:async' show AsyncError, Completer hide ControllerCallback, ControllerCancelCallback;
+import 'dart:async' show AsyncError, Completer;
+import 'dart:async' hide ControllerCallback, ControllerCancelCallback;
+import "package:boolean_selector/boolean_selector.dart";
+import '''package:charcode/ascii.dart''' as ascii;
+import """package:collection/algorithms.dart""" as algo;
+import 'dart:collection' show HashMap;
+import 'dart:math' hide ln10 show ln2, cos, sin;

pmd-dart/src/test/resources/net/sourceforge/pmd/cpd/increment.dart

@@ -0,0 +1,24 @@
+library mylibrary;
+
+var x = 0;
+
+void increment1() { x += 1; }
+void increment2() { x += 1; }
+void increment3() { x += 1; }
+void increment4() { x += 1; }
+void increment5() { x += 1; }
+void increment6() { x += 1; }
+void increment7() { x += 1; }
+void increment8() { x += 1; }
+void increment9() { x += 1; }
+void increment10() { x += 1; }
+void increment11() { x += 1; }
+void increment12() { x += 1; }
+void increment13() { x += 1; }
+void increment14() { x += 1; }
+void increment15() { x += 1; }
+void increment16() { x += 1; }
+void increment17() { x += 1; }
+void increment18() { x += 1; }
+void increment19() { x += 1; }
+void increment20() { x += 1; }

pmd-dist/pom.xml

@@ -107,6 +107,11 @@
             <artifactId>pmd-cs</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>net.sourceforge.pmd</groupId>
+            <artifactId>pmd-dart</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>net.sourceforge.pmd</groupId>
             <artifactId>pmd-fortran</artifactId>

pmd-dist/src/test/java/net/sourceforge/pmd/it/BinaryDistributionIT.java

@@ -112,7 +112,7 @@ public class BinaryDistributionIT {
 
         result = CpdExecutor.runCpd(tempDir, "-h");
 
-        result.assertExecutionResult(1, "Supported languages: [apex, cpp, cs, ecmascript, fortran, go, groovy, java, jsp, kotlin, matlab, objectivec, perl, php, plsql, python, ruby, scala, swift, vf]");
+        result.assertExecutionResult(1, "Supported languages: [apex, cpp, cs, dart, ecmascript, fortran, go, groovy, java, jsp, kotlin, matlab, objectivec, perl, php, plsql, python, ruby, scala, swift, vf]");
 
         result = CpdExecutor.runCpd(tempDir, "--minimum-tokens", "10", "--format", "text", "--files", srcDir);
         result.assertExecutionResult(4, "Found a 10 line (55 tokens) duplication in the following files:");

pom.xml

@@ -1130,6 +1130,7 @@ Additionally it includes CPD, the copy-paste-detector. CPD finds duplicated code
         <module>pmd-core</module>
         <module>pmd-cpp</module>
         <module>pmd-cs</module>
+        <module>pmd-dart</module>
         <module>pmd-dist</module>
         <module>pmd-fortran</module>
         <module>pmd-go</module>