Add tests for text access

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java

@@ -4,6 +4,8 @@
 
 package net.sourceforge.pmd.lang.ast;
 
+import net.sourceforge.pmd.lang.ast.xpath.NoAttribute;
+
 /**
  * Refinement of {@link Node} for nodes that can provide the underlying
  * source text.
@@ -29,6 +31,7 @@ public interface TextAvailableNode extends Node {
      * particular, for a {@link RootNode}, returns the whole text
      * of the file.
      */
+    @NoAttribute
     CharSequence getText();
 
 

pmd-java/etc/grammar/Java.jjt

@@ -2698,7 +2698,11 @@ void BlockStatement() #void:
 | LOOKAHEAD({ jdkVersion >= 13 && isKeyword("yield") }) YieldStatement()
 |
   LOOKAHEAD(( "final" | Annotation() )* Type() <IDENTIFIER>)
-  LocalVariableDeclaration() ";" { ((AbstractJavaNode) jjtree.peekNode()).shiftTokens(0, 1); }
+  LocalVariableDeclaration() ";" {
+      // make it so that the LocalVariableDeclaration's last token is the semicolon
+      AbstractJavaNode top = (AbstractJavaNode) jjtree.peekNode();
+      top.jjtSetLastToken(getToken(0));
+  } {}
 |
   // we need to lookahead until the "class" token,
   // because a method ref may be annotated

pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java

@@ -9,6 +9,10 @@ import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaCharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
 
+/**
+ * Support methods for the token manager. The call to {@link #newToken(int, CharStream)}
+ * is hacked in via search/replace on {@link JavaParserTokenManager}.
+ */
 final class JavaTokenFactory {
 
     private JavaTokenFactory() {

pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/JavaTextAccessTest.kt

@@ -0,0 +1,95 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.java.ast
+
+import io.kotlintest.shouldBe
+import net.sourceforge.pmd.lang.ast.TextAvailableNode
+import net.sourceforge.pmd.lang.ast.test.shouldBe
+import net.sourceforge.pmd.lang.java.ast.ASTPrimitiveType.PrimitiveType.INT
+import net.sourceforge.pmd.lang.java.ast.ParserTestCtx.Companion.StatementParsingCtx
+
+// Use a string for comparison because CharSequence are not necessarily
+// equatable to string
+private val TextAvailableNode.textStr: String get() = text.toString()
+
+class JavaTextAccessTest : ParserTestSpec({
+
+
+    parserTest("Test parens") {
+
+        inContext(StatementParsingCtx) {
+            // we use a statement context to avoid the findFirstNodeOnStraightLine skipping parentheses
+
+            "int a = ((3));" should matchStmt<ASTLocalVariableDeclaration> {
+
+                it.textStr shouldBe "int a = ((3));"
+
+                primitiveType(INT) {
+                    it.textStr shouldBe "int"
+                }
+                variableDeclarator("a") {
+                    it.textStr shouldBe "a = ((3))"
+
+                    it::getInitializer shouldBe int(3) {
+                        it.textStr shouldBe "((3))"
+                    }
+                }
+            }
+
+            "int a = ((a)).f;" should matchStmt<ASTLocalVariableDeclaration> {
+
+                it.textStr shouldBe "int a = ((a)).f;"
+
+                primitiveType(INT) {
+                    it.textStr shouldBe "int"
+                }
+                variableDeclarator("a") {
+                    it.textStr shouldBe "a = ((a)).f"
+
+                    it::getInitializer shouldBe fieldAccess("f") {
+                        it.textStr shouldBe "((a)).f"
+
+                        it::getQualifier shouldBe variableAccess("a") {
+                            it.textStr shouldBe "((a))"
+                        }
+                    }
+                }
+            }
+
+            // the left parens shouldn't be flattened by AbstractLrBinaryExpr
+            "int a = ((1 + 2) + f);" should matchStmt<ASTLocalVariableDeclaration> {
+
+                it.textStr shouldBe "int a = ((1 + 2) + f);"
+
+                primitiveType(INT) {
+                    it.textStr shouldBe "int"
+                }
+
+                variableDeclarator("a") {
+                    it.textStr shouldBe "a = ((1 + 2) + f)"
+
+                    it::getInitializer shouldBe additiveExpr(BinaryOp.ADD) {
+                        it.textStr shouldBe "((1 + 2) + f)"
+
+                        additiveExpr(BinaryOp.ADD) {
+                            it.textStr shouldBe "(1 + 2)"
+
+                            int(1) {
+                                it.textStr shouldBe "1"
+                            }
+                            int(2) {
+                                it.textStr shouldBe "2"
+                            }
+                        }
+
+                        variableAccess("f") {
+                            it.textStr shouldBe "f"
+                        }
+                    }
+                }
+            }
+        }
+    }
+})