Merge branch 'pr-2388'

[lang-test] Use tree dumps for regression tests #2388

pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TextTreeRenderer.java

@@ -0,0 +1,210 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.util.treeexport;
+
+import java.io.IOException;
+
+import net.sourceforge.pmd.annotation.Experimental;
+import net.sourceforge.pmd.lang.ast.Node;
+import net.sourceforge.pmd.properties.AbstractPropertySource;
+import net.sourceforge.pmd.properties.PropertyDescriptor;
+import net.sourceforge.pmd.properties.PropertyFactory;
+import net.sourceforge.pmd.properties.PropertySource;
+
+/**
+ * A simple recursive printer. Output looks like so:
+ *
+ * <pre>
+ *
+ * +- LocalVariableDeclaration
+ *    +- Type
+ *    |  +- PrimitiveType
+ *    +- VariableDeclarator
+ *       +- VariableDeclaratorId
+ *       +- VariableInitializer
+ *          +- 1 child not shown
+ *
+ * </pre>
+ *
+ * or
+ *
+ * <pre>
+ *
+ * └─ LocalVariableDeclaration
+ *    ├─ Type
+ *    │  └─ PrimitiveType
+ *    └─ VariableDeclarator
+ *       ├─ VariableDeclaratorId
+ *       └─ VariableInitializer
+ *          └─ 1 child not shown
+ *
+ * </pre>
+ *
+ *
+ * By default just prints the structure, like shown above. You can
+ * configure it to render nodes differently by overriding {@link #appendNodeInfoLn(Appendable, Node)}.
+ */
+@Experimental
+public class TextTreeRenderer implements TreeRenderer {
+
+    static final TreeRendererDescriptor DESCRIPTOR = new TreeRendererDescriptor() {
+
+        private final PropertyDescriptor<Boolean> onlyAscii =
+            PropertyFactory.booleanProperty("onlyAsciiChars")
+                           .defaultValue(false)
+                           .desc("Use only ASCII characters in the structure")
+                           .build();
+
+        private final PropertyDescriptor<Integer> maxLevel =
+            PropertyFactory.intProperty("maxLevel")
+                           .defaultValue(-1)
+                           .desc("Max level on which to recurse. Negative means unbounded")
+                           .build();
+
+        @Override
+        public PropertySource newPropertyBundle() {
+
+            PropertySource bundle = new AbstractPropertySource() {
+                @Override
+                protected String getPropertySourceType() {
+                    return "tree renderer";
+                }
+
+                @Override
+                public String getName() {
+                    return "text";
+                }
+            };
+
+            bundle.definePropertyDescriptor(onlyAscii);
+            bundle.definePropertyDescriptor(maxLevel);
+
+            return bundle;
+        }
+
+        @Override
+        public String id() {
+            return "text";
+        }
+
+        @Override
+        public String description() {
+            return "Text renderer";
+        }
+
+        @Override
+        public TreeRenderer produceRenderer(PropertySource properties) {
+            return new TextTreeRenderer(properties.getProperty(onlyAscii), properties.getProperty(maxLevel));
+        }
+    };
+
+    private final Strings str;
+    private final int maxLevel;
+
+    /**
+     * Creates a new text renderer.
+     *
+     * @param onlyAscii Whether to output the skeleton of the tree with
+     *                  only ascii characters. If false, uses unicode chars
+     *                  like '├'
+     * @param maxLevel  Max level on which to recurse. Negative means
+     *                  unbounded. If the max level is reached, a placeholder
+     *                  is dumped, like "1 child is not shown". This is
+     *                  controlled by {@link #appendBoundaryForNodeLn(Node, Appendable, String)}.
+     */
+    public TextTreeRenderer(boolean onlyAscii, int maxLevel) {
+        this.str = onlyAscii ? Strings.ASCII : Strings.UNICODE;
+        this.maxLevel = maxLevel;
+    }
+
+    @Override
+    public void renderSubtree(Node node, Appendable out) throws IOException {
+        printInnerNode(node, out, 0, "", true);
+    }
+
+    private String childPrefix(String prefix, boolean isTail) {
+        return prefix + (isTail ? str.gap : str.verticalEdge);
+    }
+
+
+    protected final void appendIndent(Appendable out, String prefix, boolean isTail) throws IOException {
+        out.append(prefix).append(isTail ? str.tailFork : str.fork);
+    }
+
+    /**
+     * Append info about the node. The indent has already been appended.
+     * This should end with a newline. The default just appends the name
+     * of the node, and no other information.
+     */
+    protected void appendNodeInfoLn(Appendable out, Node node) throws IOException {
+        out.append(node.getXPathNodeName()).append("\n");
+    }
+
+
+    private void printInnerNode(Node node,
+                                Appendable out,
+                                int level,
+                                String prefix,
+                                boolean isTail) throws IOException {
+
+        appendIndent(out, prefix, isTail);
+        appendNodeInfoLn(out, node);
+
+        if (level == maxLevel) {
+            if (node.getNumChildren() > 0) {
+                appendBoundaryForNodeLn(node, out, childPrefix(prefix, isTail));
+            }
+        } else {
+            int n = node.getNumChildren() - 1;
+            String childPrefix = childPrefix(prefix, isTail);
+            for (int i = 0; i < node.getNumChildren(); i++) {
+                Node child = node.getChild(i);
+                printInnerNode(child, out, level + 1, childPrefix, i == n);
+            }
+        }
+    }
+
+    protected void appendBoundaryForNodeLn(Node node, Appendable out, String indentStr) throws IOException {
+        appendIndent(out, indentStr, true);
+
+        if (node.getNumChildren() == 1) {
+            out.append("1 child is not shown");
+        } else {
+            out.append(String.valueOf(node.getNumChildren())).append(" children are not shown");
+        }
+
+        out.append('\n');
+    }
+
+    private static final class Strings {
+
+        private static final Strings ASCII = new Strings(
+            "+- ",
+            "+- ",
+            "|  ",
+            "   "
+        );
+        private static final Strings UNICODE = new Strings(
+            "└─ ",
+            "├─ ",
+            "│  ",
+            "   "
+        );
+
+        private final String tailFork;
+        private final String fork;
+        private final String verticalEdge;
+        private final String gap;
+
+
+        private Strings(String tailFork, String fork, String verticalEdge, String gap) {
+            this.tailFork = tailFork;
+            this.fork = fork;
+            this.verticalEdge = verticalEdge;
+            this.gap = gap;
+        }
+    }
+
+}

pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeRenderers.java

@@ -102,7 +102,10 @@ public final class TreeRenderers {
 
 
     static {
-        REGISTRY.put(XML.id(), XML);
+        List<TreeRendererDescriptor> builtinDescriptors = Arrays.asList(XML, TextTreeRenderer.DESCRIPTOR);
+        for (TreeRendererDescriptor descriptor : builtinDescriptors) {
+            REGISTRY.put(descriptor.id(), descriptor);
+        }
     }
 
 

pmd-lang-test/pom.xml

@@ -132,7 +132,7 @@
         <dependency>
             <groupId>com.github.oowekyala.treeutils</groupId>
             <artifactId>tree-matchers</artifactId>
-            <version>2.0.1</version>
+            <version>2.1.0</version>
             <scope>compile</scope>
         </dependency>
 

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/lang/ast/test/AstMatcherDslAdapter.kt

@@ -4,6 +4,7 @@
 
 package net.sourceforge.pmd.lang.ast.test
 
+import com.github.oowekyala.treeutils.DoublyLinkedTreeLikeAdapter
 import com.github.oowekyala.treeutils.TreeLikeAdapter
 import com.github.oowekyala.treeutils.matchers.MatchingConfig
 import com.github.oowekyala.treeutils.matchers.TreeNodeWrapper
@@ -12,10 +13,14 @@ import com.github.oowekyala.treeutils.printers.KotlintestBeanTreePrinter
 import net.sourceforge.pmd.lang.ast.Node
 
 /** An adapter for [baseShouldMatchSubtree]. */
-object NodeTreeLikeAdapter : TreeLikeAdapter<Node> {
+object NodeTreeLikeAdapter : DoublyLinkedTreeLikeAdapter<Node> {
     override fun getChildren(node: Node): List<Node> = node.findChildrenOfType(Node::class.java)
 
     override fun nodeName(type: Class<out Node>): String = type.simpleName.removePrefix("AST")
+
+    override fun getParent(node: Node): Node? = node.parent
+
+    override fun getChild(node: Node, index: Int): Node? = node.safeGetChild(index)
 }
 
 /** A subtree matcher written in the DSL documented on [TreeNodeWrapper]. */

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/lang/ast/test/BaseParsingHelper.kt

@@ -35,12 +35,18 @@ abstract class BaseParsingHelper<Self : BaseParsingHelper<Self, T>, T : RootNode
 
             @JvmStatic
             val defaultNoProcess = Params(false, null, null, "")
+
             @JvmStatic
             val defaultProcess = Params(true, null, null, "")
 
         }
     }
 
+    internal val resourceLoader: Class<*>
+        get() = params.resourceLoader ?: javaClass
+
+    internal val resourcePrefix: String get() = params.resourcePrefix
+
     /**
      * Returns the language version with the given version string.
      * If null, this defaults to the default language version for
@@ -138,10 +144,10 @@ abstract class BaseParsingHelper<Self : BaseParsingHelper<Self, T>, T : RootNode
             parse(readClassSource(clazz), version)
 
     protected fun readResource(resourceName: String): String {
-        val rloader = params.resourceLoader ?: javaClass
 
-        val input = rloader.getResourceAsStream(params.resourcePrefix + resourceName)
-                ?: throw IllegalArgumentException("Unable to find resource file ${params.resourcePrefix + resourceName} from $rloader")
+        val input = resourceLoader.getResourceAsStream(params.resourcePrefix + resourceName)
+                ?: throw IllegalArgumentException("Unable to find resource file ${params.resourcePrefix + resourceName} from $resourceLoader")
+
         return consume(input)
     }
 

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/lang/ast/test/BaseTreeDumpTest.kt

@@ -0,0 +1,74 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ast.test
+
+import net.sourceforge.pmd.util.treeexport.TreeRenderer
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlin.test.assertEquals
+
+
+/**
+ * Compare a dump of a file against a saved baseline.
+ *
+ * @param printer The node printer used to dump the trees
+ * @param extension Extension that the unparsed source file is supposed to have
+ */
+abstract class BaseTreeDumpTest(
+        val printer: TreeRenderer,
+        val extension: String
+) {
+
+    abstract val parser: BaseParsingHelper<*, *>
+
+    /**
+     * Executes the test. The test files are looked up using the [parser].
+     * The reference test file must be named [fileBaseName] + [ExpectedExt].
+     * The source file to parse must be named [fileBaseName] + [extension].
+     */
+    fun doTest(fileBaseName: String) {
+        val expectedFile = findTestFile(parser.resourceLoader, "${parser.resourcePrefix}/$fileBaseName$ExpectedExt").toFile()
+        val sourceFile = findTestFile(parser.resourceLoader, "${parser.resourcePrefix}/$fileBaseName$extension").toFile()
+
+        assert(sourceFile.isFile) {
+            "Source file $sourceFile is missing"
+        }
+
+        val parsed = parser.parse(sourceFile.readText()) // UTF-8
+        val actual = StringBuilder().also { printer.renderSubtree(parsed, it) }.toString()
+
+        if (!expectedFile.exists()) {
+            expectedFile.writeText(actual)
+            throw AssertionError("Reference file doesn't exist, created it at $expectedFile")
+        }
+
+        val expected = expectedFile.readText()
+
+        assertEquals(expected.normalize(), actual.normalize(), "Tree dump comparison failed, see the reference: $expectedFile")
+    }
+
+    // Outputting a path makes for better error messages
+    private val srcTestResources = let {
+        // this is set from maven surefire
+        System.getProperty("mvn.project.src.test.resources")
+                ?.let { Paths.get(it).toAbsolutePath() }
+                // that's for when the tests are run inside the IDE
+                ?: Paths.get(javaClass.protectionDomain.codeSource.location.file)
+                        // go up from target/test-classes into the project root
+                        .resolve("../../src/test/resources").normalize()
+    }
+
+    private fun findTestFile(contextClass: Class<*>, resourcePath: String): Path {
+        val path = contextClass.`package`.name.replace('.', '/')
+        return srcTestResources.resolve("$path/$resourcePath")
+    }
+
+    companion object {
+        const val ExpectedExt = ".txt"
+
+        fun String.normalize() = replace(Regex("\\R"), "\n")
+    }
+
+}

pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/lang/ast/test/NodePrinters.kt

@@ -0,0 +1,77 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ast.test
+
+import net.sourceforge.pmd.lang.ast.Node
+import net.sourceforge.pmd.lang.ast.xpath.Attribute
+import net.sourceforge.pmd.util.treeexport.TextTreeRenderer
+import org.apache.commons.lang3.StringEscapeUtils
+
+/**
+ * Prints just the structure, like so:
+ *
+ *           └── LocalVariableDeclaration
+ *               ├── Type
+ *               │   └── PrimitiveType
+ *               └── VariableDeclarator
+ *                   ├── VariableDeclaratorId
+ *                   └── VariableInitializer
+ *                       └── 1 child not shown
+ *
+ */
+val SimpleNodePrinter = TextTreeRenderer(true, -1)
+
+
+open class RelevantAttributePrinter : BaseNodeAttributePrinter() {
+
+    private val Ignored = setOf("BeginLine", "EndLine", "BeginColumn", "EndColumn", "FindBoundary", "SingleLine")
+
+    override fun ignoreAttribute(node: Node, attribute: Attribute): Boolean =
+            Ignored.contains(attribute.name) || attribute.name == "Image" && attribute.value == null
+
+}
+
+/**
+ * Base attribute printer, subclass to filter attributes.
+ */
+open class BaseNodeAttributePrinter : TextTreeRenderer(true, -1) {
+
+    protected open fun ignoreAttribute(node: Node, attribute: Attribute): Boolean = true
+
+    override fun appendNodeInfoLn(out: Appendable, node: Node) {
+        out.append(node.xPathNodeName)
+
+        node.xPathAttributesIterator
+            .asSequence()
+            // sort to get deterministic results
+            .sortedBy { it.name }
+            .filterNot { ignoreAttribute(node, it) }
+            .joinTo(buffer = out, prefix = "[", postfix = "]") {
+                "@${it.name} = ${valueToString(it.value)}"
+            }
+    }
+
+    protected open fun valueToString(value: Any?): String? {
+        return when (value) {
+            is String -> "\"" + StringEscapeUtils.unescapeJava(value) + "\""
+            is Char -> '\''.toString() + value.toString().replace("'".toRegex(), "\\'") + '\''.toString()
+            is Enum<*> -> value.enumDeclaringClass.simpleName + "." + value.name
+            is Class<*> -> value.canonicalName?.let { "$it.class" }
+            is Number, is Boolean, null -> value.toString()
+            else -> null
+        }
+    }
+
+    private val Enum<*>.enumDeclaringClass: Class<*>
+        get() = this.javaClass.let {
+            when {
+                it.isEnum -> it
+                else -> it.enclosingClass.takeIf { it.isEnum }
+                        ?: throw IllegalStateException()
+            }
+        }
+
+}
+

pmd-scala/src/test/kotlin/net/sourceforge/pmd/lang/scala/ast/ScalaParserTests.kt

@@ -0,0 +1,23 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.scala.ast
+
+import net.sourceforge.pmd.lang.ast.test.BaseParsingHelper
+import net.sourceforge.pmd.lang.ast.test.BaseTreeDumpTest
+import net.sourceforge.pmd.lang.ast.test.SimpleNodePrinter
+import org.junit.Test
+
+class ScalaParserTests : BaseTreeDumpTest(SimpleNodePrinter, ".scala") {
+
+    override val parser: BaseParsingHelper<*, *>
+        get() = ScalaParsingHelper.DEFAULT.withResourceContext(javaClass, "testdata")
+
+    @Test
+    fun testSomeScalaFeatures() = doTest("List")
+
+    @Test
+    fun testPackageObject() = doTest("package")
+
+}

pmd-scala/src/test/resources/net/sourceforge/pmd/lang/scala/ast/testdata/package.scala

@@ -0,0 +1,22 @@
+
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package scala.collection
+
+
+package object immutable {
+  type StringOps = scala.collection.StringOps
+  val StringOps = scala.collection.StringOps
+  type StringView = scala.collection.StringView
+  val StringView = scala.collection.StringView
+
+  @deprecated("Use Iterable instead of Traversable", "2.13.0")
+  type Traversable[+X] = Iterable[X]
+  @deprecated("Use Iterable instead of Traversable", "2.13.0")
+  val Traversable = Iterable
+
+  @deprecated("Use Map instead of DefaultMap", "2.13.0")
+  type DefaultMap[K, +V] = scala.collection.immutable.Map[K, V]
+}

pmd-scala/src/test/resources/net/sourceforge/pmd/lang/scala/ast/testdata/package.txt

@@ -0,0 +1,90 @@
++- Source
+   +- Pkg
+      +- TermSelect
+      |  +- TermName
+      |  +- TermName
+      +- PkgObject
+         +- TermName
+         +- Template
+            +- Self
+            |  +- NameAnonymous
+            +- DefnType
+            |  +- TypeName
+            |  +- TypeSelect
+            |     +- TermSelect
+            |     |  +- TermName
+            |     |  +- TermName
+            |     +- TypeName
+            +- DefnVal
+            |  +- PatVar
+            |  |  +- TermName
+            |  +- TermSelect
+            |     +- TermSelect
+            |     |  +- TermName
+            |     |  +- TermName
+            |     +- TermName
+            +- DefnType
+            |  +- TypeName
+            |  +- TypeSelect
+            |     +- TermSelect
+            |     |  +- TermName
+            |     |  +- TermName
+            |     +- TypeName
+            +- DefnVal
+            |  +- PatVar
+            |  |  +- TermName
+            |  +- TermSelect
+            |     +- TermSelect
+            |     |  +- TermName
+            |     |  +- TermName
+            |     +- TermName
+            +- DefnType
+            |  +- ModAnnot
+            |  |  +- Init
+            |  |     +- TypeName
+            |  |     +- NameAnonymous
+            |  |     +- LitString
+            |  |     +- LitString
+            |  +- TypeName
+            |  +- TypeParam
+            |  |  +- ModCovariant
+            |  |  +- TypeName
+            |  |  +- TypeBounds
+            |  +- TypeApply
+            |     +- TypeName
+            |     +- TypeName
+            +- DefnVal
+            |  +- ModAnnot
+            |  |  +- Init
+            |  |     +- TypeName
+            |  |     +- NameAnonymous
+            |  |     +- LitString
+            |  |     +- LitString
+            |  +- PatVar
+            |  |  +- TermName
+            |  +- TermName
+            +- DefnType
+               +- ModAnnot
+               |  +- Init
+               |     +- TypeName
+               |     +- NameAnonymous
+               |     +- LitString
+               |     +- LitString
+               +- TypeName
+               +- TypeParam
+               |  +- TypeName
+               |  +- TypeBounds
+               +- TypeParam
+               |  +- ModCovariant
+               |  +- TypeName
+               |  +- TypeBounds
+               +- TypeApply
+                  +- TypeSelect
+                  |  +- TermSelect
+                  |  |  +- TermSelect
+                  |  |  |  +- TermName
+                  |  |  |  +- TermName
+                  |  |  +- TermName
+                  |  +- TypeName
+                  +- TypeName
+                  +- TypeName

pom.xml

@@ -256,6 +256,9 @@
                     <configuration>
                         <forkMode>once</forkMode>
                         <runOrder>alphabetical</runOrder>
+                        <systemPropertyVariables>
+                            <mvn.project.src.test.resources>${project.basedir}/src/test/resources</mvn.project.src.test.resources>
+                        </systemPropertyVariables>
                     </configuration>
                 </plugin>
                 <plugin>