[java] Support String Templates for Java 21 Preview

JEP 430
This commit is contained in:
Andreas Dangel
2023-08-04 17:13:48 +02:00
parent c2710c9a91
commit 715d58fef3
9 changed files with 1237 additions and 2 deletions

View File

@ -1,4 +1,6 @@
/**
* Support "JEP 430: String Templates" for Java 21 Preview.
* New AST nodes: ASTTemplateExpression, ASTTemplate, ASTTemplateFragment
* Promote "JEP 441: Pattern Matching for switch" as permanent language feature for Java 21.
* Renamed SwitchGuard to Guard.
* Promote "JEP 440: Record Patterns" as permanent language feature for Java 21.
@ -584,6 +586,53 @@ PARSER_END(JavaParserImpl)
TOKEN_MGR_DECLS :
{
protected List<JavaComment> comments = new ArrayList<JavaComment>();
enum TokenContext { STRING_TEMPLATE, TEXT_BLOCK_TEMPLATE, BLOCK; }
private static final java.util.regex.Pattern TEXT_BLOCK_TEMPLATE_END_PATTERN =
java.util.regex.Pattern.compile("^}[^\"]*\"\"\"");
private static final java.util.regex.Pattern STRING_TEMPLATE_MID_OR_END_PATTERN =
java.util.regex.Pattern.compile("^}(?:[^\"\\\\\n\r]|\\\\(?:[ntbrfs\\\\'\"]|[0-7][0-7]?|[0-3][0-7][0-7]))*(\\{|\")");
private TokenContext determineContext() {
Throwable t = new Throwable().fillInStackTrace();
for (StackTraceElement e : t.getStackTrace()) {
String method = e.getMethodName();
if ("TextBlockTemplate".equals(method)) {
return TokenContext.TEXT_BLOCK_TEMPLATE;
} else if ("StringTemplate".equals(method)) {
return TokenContext.STRING_TEMPLATE;
} else if ("Block".equals(method)
|| "ClassOrInterfaceBody".equals(method)
|| "ArrayInitializer".equals(method)
|| "MemberValueArrayInitializer".equals(method)
|| "SwitchBlock".equals(method)) {
return TokenContext.BLOCK;
}
}
return null;
}
private net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken rereadTokenAs(int kind, int length) {
input_stream.backup(lengthOfMatch);
try {
for (int i = 0; i < length; i++) {
input_stream.readChar();
}
} catch (java.io.EOFException eofException) {
throw new IllegalStateException(eofException);
}
jjmatchedKind = kind;
return jjFillToken();
}
private net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken handleBlock() {
net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken matchedToken = rereadTokenAs(JavaTokenKinds.RBRACE, 1);
if (!"}".equals(input_stream.getTokenImage())) {
throw new IllegalStateException("Expected '}'");
}
return matchedToken;
}
}
/* WHITE SPACE */
@ -754,7 +803,8 @@ TOKEN :
| < #EXPONENT_TAIL: (["+","-"])? <DIGIT_SEQ> >
| < CHARACTER_LITERAL: "'" ( ~["'", "\\","\n","\r"] | <STRING_ESCAPE> ) "'" >
| < STRING_LITERAL: "\"" ( ~["\"","\\","\n","\r"] | <STRING_ESCAPE> )* "\"" >
| < STRING_LITERAL: "\"" (<STRING_CHARACTER>)* "\"" >
| < #STRING_CHARACTER: ~["\"","\\","\n","\r"] | <STRING_ESCAPE> >
| < #STRING_ESCAPE:
"\\"
( ["n","t","b","r","f","s","\\","'","\""]
@ -763,9 +813,12 @@ TOKEN :
| ["0"-"3"] ["0"-"7"] ["0"-"7"]
)
>
| < #TEXT_BLOCK_CHARACTER: ~["\\"] | <STRING_ESCAPE> | ("\\")? <LINE_TERMINATOR> >
}
/* TEXT BLOCKS */
// note: Text Blocks need an own lexical state, so that we can reliably determine
// the end of the text block (""") which is 3 characters long.
MORE :
{
< "\"\"\"" (<HORIZONTAL_WHITESPACE>)* <LINE_TERMINATOR> > : IN_TEXT_BLOCK_LITERAL
@ -780,7 +833,7 @@ TOKEN :
<IN_TEXT_BLOCK_LITERAL>
MORE :
{
< ~["\\"] | <STRING_ESCAPE> | ("\\")? <LINE_TERMINATOR> >
< <TEXT_BLOCK_CHARACTER> >
}
/* IDENTIFIERS */
@ -982,6 +1035,87 @@ TOKEN :
| < GT: ">" >
}
/* FRAGMENTS */
// Note: The fragments introduce ambiguity with other token productions, especially the separator token "}" (RBRACE).
// In order to produce the correct token sequence, the ambiguity needs to be resolved using the context.
// That means, that STRING_TEMPLATE_MID/END and TEXT_BLOCK_TEMPLATE_MID/END could actually be a closing bracket ("}").
// Additionally, a STRING_TEMPLATE_MID could be a TEXT_BLOCK_TEMPLATE_MID and the other way round.
// See JLS 3.13 Fragments (Java 21 Preview)
TOKEN :
{
< STRING_TEMPLATE_BEGIN: "\"" <STRING_FRAGMENT> "\\{" >
| < STRING_TEMPLATE_MID: "}" <STRING_FRAGMENT> "\\{" >
{
{
TokenContext ctx = determineContext();
switch (ctx) {
case TEXT_BLOCK_TEMPLATE:
jjmatchedKind = TEXT_BLOCK_TEMPLATE_MID;
matchedToken = jjFillToken();
break;
case BLOCK:
matchedToken = handleBlock();
break;
}
}
}
| < STRING_TEMPLATE_END: "}" <STRING_FRAGMENT> "\"" >
{
{
TokenContext ctx = determineContext();
if (ctx == TokenContext.BLOCK) {
matchedToken = handleBlock();
}
}
}
| < #STRING_FRAGMENT: (<STRING_CHARACTER>)* >
| < TEXT_BLOCK_TEMPLATE_BEGIN: "\"\"\"" (<HORIZONTAL_WHITESPACE>)* <LINE_TERMINATOR> <TEXT_BLOCK_FRAGMENT> "\\{" >
| < TEXT_BLOCK_TEMPLATE_MID: "}" <TEXT_BLOCK_FRAGMENT> "\\{" >
{
{
TokenContext ctx = determineContext();
switch (ctx) {
case STRING_TEMPLATE: {
java.util.regex.Matcher m = STRING_TEMPLATE_MID_OR_END_PATTERN.matcher(matchedToken.getImage());
if (m.find()) {
int kind = STRING_TEMPLATE_END;
if ("\\{".equals(m.group(1))) {
kind = STRING_TEMPLATE_MID;
}
matchedToken = rereadTokenAs(kind, m.end());
}
break;
}
case TEXT_BLOCK_TEMPLATE: {
// Note: TEXT_BLOCK_FRAGMENT is not really correct and might match """ as part of TEXT_BLOCK_TEMPLATE_MID
// instead of TEXT_BLOCK_TEMPLATE_END. In case this happens, this is corrected here.
java.util.regex.Matcher m = TEXT_BLOCK_TEMPLATE_END_PATTERN.matcher(matchedToken.getImage());
if (m.find()) {
matchedToken = rereadTokenAs(TEXT_BLOCK_TEMPLATE_END, m.end());
}
break;
}
case BLOCK:
matchedToken = handleBlock();
break;
}
}
}
| < TEXT_BLOCK_TEMPLATE_END: "}" <TEXT_BLOCK_FRAGMENT> "\"\"\"" >
{
{
TokenContext ctx = determineContext();
if (ctx == TokenContext.BLOCK) {
matchedToken = handleBlock();
}
}
}
| < #TEXT_BLOCK_FRAGMENT: (<TEXT_BLOCK_CHARACTER>)* >
}
/*****************************************
* THE JAVA LANGUAGE GRAMMAR STARTS HERE *
*****************************************/
@ -2031,6 +2165,7 @@ void PrimaryStep2() #void:
// "super" alone is not a valid expression
("." MemberSelector() | MethodReference())
| MemberSelector()
| {forceExprContext();} TemplateArgument() #TemplateExpression(2)
)
// catches the case where the ambig name is the start of an array type
| LOOKAHEAD("@" | "[" "]") {forceTypeContext();} Dims() #ArrayType(2) (MethodReference() | "." "class" #ClassLiteral(1))
@ -2135,6 +2270,45 @@ boolean LambdaParameterType() #void :
| FormalParamType() { return false; }
}
void TemplateArgument() #void :
{}
{
Template()
| StringLiteral()
}
void Template() :
{}
{
StringTemplate()
| TextBlockTemplate()
}
void StringTemplate() #void :
{}
{
<STRING_TEMPLATE_BEGIN> { setLastTokenImage(jjtThis); } #TemplateFragment
EmbeddedExpression()
( <STRING_TEMPLATE_MID> { setLastTokenImage(jjtThis); } #TemplateFragment EmbeddedExpression() )*
<STRING_TEMPLATE_END> { setLastTokenImage(jjtThis); } #TemplateFragment
}
void TextBlockTemplate() #void :
{}
{
<TEXT_BLOCK_TEMPLATE_BEGIN> { setLastTokenImage(jjtThis); } #TemplateFragment
EmbeddedExpression()
( <TEXT_BLOCK_TEMPLATE_MID> { setLastTokenImage(jjtThis); } #TemplateFragment EmbeddedExpression() )*
<TEXT_BLOCK_TEMPLATE_END> { setLastTokenImage(jjtThis); } #TemplateFragment
}
void EmbeddedExpression() #void :
{}
{
[ Expression() ]
}
void Literal() #void :
{}
{

View File

@ -0,0 +1,30 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.java.ast;
import net.sourceforge.pmd.annotation.Experimental;
/**
* This is a Java 21 Preview feature.
*
* <pre class="grammar">
*
* Template ::= ({@link ASTTemplateFragment TemplateFragment} {@link ASTExpression Expression}?)* {@link ASTTemplateFragment TemplateFragment}
*
* </pre>
*
* @see <a href="https://openjdk.org/jeps/430">JEP 430: String Templates (Preview)</a>
*/
@Experimental
public final class ASTTemplate extends AbstractJavaNode {
ASTTemplate(int i) {
super(i);
}
@Override
protected <P, R> R acceptVisitor(JavaVisitor<? super P, ? extends R> visitor, P data) {
return visitor.visit(this, data);
}
}

View File

@ -0,0 +1,50 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.java.ast;
import net.sourceforge.pmd.annotation.Experimental;
import net.sourceforge.pmd.lang.java.ast.ASTAssignableExpr.ASTNamedReferenceExpr;
/**
* This is a Java 21 Preview feature.
*
* <pre class="grammar">
*
* TemplateExpression ::= ({@link ASTVariableAccess VariableAccess} | {@link ASTFieldAccess FieldAccess})
* ({@link ASTTemplate Template} | {@link ASTStringLiteral StringLiteral})
*
* </pre>
*
* @see <a href="https://openjdk.org/jeps/430">JEP 430: String Templates (Preview)</a>
*/
@Experimental
public final class ASTTemplateExpression extends AbstractJavaExpr {
ASTTemplateExpression(int i) {
super(i);
}
@Override
protected <P, R> R acceptVisitor(JavaVisitor<? super P, ? extends R> visitor, P data) {
return visitor.visit(this, data);
}
public ASTExpression getTemplateProcessor() {
return (ASTExpression) getChild(0);
}
public JavaNode getTemplateArgument() {
return getChild(1);
}
public boolean isStringTemplate() {
String name;
if (getTemplateProcessor() instanceof ASTNamedReferenceExpr) {
name = ((ASTNamedReferenceExpr) getTemplateProcessor()).getName();
} else {
name = getTemplateProcessor().getFirstToken().getImage();
}
return "STR".equals(name);
}
}

View File

@ -0,0 +1,31 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.java.ast;
import net.sourceforge.pmd.annotation.Experimental;
/**
* This is a Java 21 Preview feature.
*
* <pre class="grammar">
*
* TemplateFragment ::= StringTemplateBegin|StringTemplateMid|StringTemplateEnd
* |TextBlockTemplateBegin|TextBlockTemplateMid|TextBlockTemplateEnd
*
* </pre>
*
* @see <a href="https://openjdk.org/jeps/430">JEP 430: String Templates (Preview)</a>
*/
@Experimental
public final class ASTTemplateFragment extends AbstractJavaNode {
ASTTemplateFragment(int i) {
super(i);
}
@Override
protected <P, R> R acceptVisitor(JavaVisitor<? super P, ? extends R> visitor, P data) {
return visitor.visit(this, data);
}
}

View File

@ -39,6 +39,7 @@ import net.sourceforge.pmd.lang.java.ast.ASTStringLiteral;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchArrowBranch;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchExpression;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchLabel;
import net.sourceforge.pmd.lang.java.ast.ASTTemplateExpression;
import net.sourceforge.pmd.lang.java.ast.ASTTryStatement;
import net.sourceforge.pmd.lang.java.ast.ASTType;
import net.sourceforge.pmd.lang.java.ast.ASTTypeArguments;
@ -164,6 +165,12 @@ public class LanguageLevelChecker<T> {
*/
DECONSTRUCTION_PATTERNS_IN_ENHANCED_FOR_STATEMENT(20, 20, false),
/**
* String Templates.
* @see <a href="https://openjdk.org/jeps/430">JEP 430: String Templates (Preview)</a>
*/
STRING_TEMPLATES(21, 21, false),
; // SUPPRESS CHECKSTYLE enum trailing semi is awesome
@ -626,6 +633,12 @@ public class LanguageLevelChecker<T> {
return null;
}
@Override
public Void visit(ASTTemplateExpression node, T data) {
check(node, PreviewFeature.STRING_TEMPLATES, data);
return null;
}
@Override
public Void visitTypeDecl(ASTAnyTypeDeclaration node, T data) {
if (node.getModifiers().hasAnyExplicitly(JModifier.SEALED, JModifier.NON_SEALED)) {

View File

@ -51,6 +51,7 @@ import net.sourceforge.pmd.lang.java.ast.ASTSuperExpression;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchExpression;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchLabel;
import net.sourceforge.pmd.lang.java.ast.ASTSwitchLike;
import net.sourceforge.pmd.lang.java.ast.ASTTemplateExpression;
import net.sourceforge.pmd.lang.java.ast.ASTThisExpression;
import net.sourceforge.pmd.lang.java.ast.ASTType;
import net.sourceforge.pmd.lang.java.ast.ASTTypeParameter;
@ -422,6 +423,15 @@ public final class LazyTypeResolver extends JavaVisitorBase<TypingContext, @NonN
return node.getCastType().getTypeMirror(ctx);
}
@Override
public @NonNull JTypeMirror visit(ASTTemplateExpression node, TypingContext data) {
if (node.isStringTemplate()) {
return stringType;
}
return ts.UNKNOWN;
}
@Override
public JTypeMirror visit(ASTNullLiteral node, TypingContext ctx) {
return ts.NULL_TYPE;

View File

@ -4,10 +4,19 @@
package net.sourceforge.pmd.lang.java.ast;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.lang.ast.ParseException;
import net.sourceforge.pmd.lang.ast.test.BaseParsingHelper;
import net.sourceforge.pmd.lang.ast.test.BaseTreeDumpTest;
import net.sourceforge.pmd.lang.ast.test.RelevantAttributePrinter;
import net.sourceforge.pmd.lang.java.JavaParsingHelper;
import net.sourceforge.pmd.lang.java.symbols.JClassSymbol;
import net.sourceforge.pmd.lang.java.types.JTypeMirror;
class Java21PreviewTreeDumpTest extends BaseTreeDumpTest {
private final JavaParsingHelper java21p =
@ -24,4 +33,22 @@ class Java21PreviewTreeDumpTest extends BaseTreeDumpTest {
return java21p;
}
@Test
void templateProcessors() {
doTest("Jep430_StringTemplates");
}
@Test
void templateProcessorsBeforeJava21Preview() {
ParseException thrown = assertThrows(ParseException.class, () -> java21.parseResource("Jep430_StringTemplates.java"));
assertTrue(thrown.getMessage().contains("String templates is a preview feature of JDK 21, you should select your language version accordingly"));
}
@Test
void templateExpressionType() {
ASTCompilationUnit unit = java21p.parse("class Foo {{ int i = 1; String s = STR.\"i = \\{i}\"; }}");
ASTTemplateExpression templateExpression = unit.descendants(ASTTemplateExpression.class).first();
JTypeMirror typeMirror = templateExpression.getTypeMirror();
assertEquals("java.lang.String", ((JClassSymbol) typeMirror.getSymbol()).getCanonicalName());
}
}

View File

@ -0,0 +1,193 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
import static java.lang.StringTemplate.RAW;
import static java.util.FormatProcessor.FMT;
import java.io.File;
import java.time.format.DateTimeFormatter;
import java.time.LocalTime;
/**
* @see <a href="https://openjdk.org/jeps/430">JEP 430: String Templates (Preview)</a>
*/
class Jep430_StringTemplates {
record Request(String date, String time, String ipAddress) {}
static void STRTemplateProcessor() {
// Embedded expressions can be strings
String firstName = "Bill";
String lastName = "Duck";
String fullName = STR."\{firstName} \{lastName}";
// | "Bill Duck"
String sortName = STR."\{lastName}, \{firstName}";
// | "Duck, Bill"
// Embedded expressions can perform arithmetic
int x = 10, y = 20;
String s1 = STR."\{x} + \{y} = \{x + y}";
// | "10 + 20 = 30"
// Embedded expressions can invoke methods and access fields
String s2 = STR."You have a \{getOfferType()} waiting for you!";
// | "You have a gift waiting for you!"
Request req = new Request("2022-03-25", "15:34", "8.8.8.8");
String t = STR."Access at \{req.date} \{req.time} from \{req.ipAddress}";
//| "Access at 2022-03-25 15:34 from 8.8.8.8"
String filePath = "tmp.dat";
File file = new File(filePath);
String old = "The file " + filePath + " " + (file.exists() ? "does" : "does not") + " exist";
String msg = STR."The file \{filePath} \{file.exists() ? "does" : "does not"} exist";
// | "The file tmp.dat does exist" or "The file tmp.dat does not exist"
// spread over multiple lines
String time = STR."The time is \{
// The java.time.format package is very useful
DateTimeFormatter
.ofPattern("HH:mm:ss")
.format(LocalTime.now())
} right now";
// | "The time is 12:34:56 right now"
// Left to right
// Embedded expressions can be postfix increment expressions
int index = 0;
String data = STR."\{index++}, \{index++}, \{index++}, \{index++}";
// | "0, 1, 2, 3"
// Embedded expression is a (nested) template expression
String[] fruit = { "apples", "oranges", "peaches" };
String s3 = STR."\{fruit[0]}, \{STR."\{fruit[1]}, \{fruit[2]}"}";
// | "apples, oranges, peaches"
String s4 = STR."\{fruit[0]}, \{
STR."\{fruit[1]}, \{fruit[2]}"
}";
}
static String getOfferType() { return "_getOfferType_"; }
static void multilineTemplateExpressions() {
String title = "My Web Page";
String text = "Hello, world";
String html = STR."""
<html>
<head>
<title>\{title}</title>
</head>
<body>
<p>\{text}</p>
</body>
</html>
""";
/*
| """
| <html>
| <head>
| <title>My Web Page</title>
| </head>
| <body>
| <p>Hello, world</p>
| </body>
| </html>
| """
*/
String name = "Joan Smith";
String phone = "555-123-4567";
String address = "1 Maple Drive, Anytown";
String json = STR."""
{
"name": "\{name}",
"phone": "\{phone}",
"address": "\{address}"
}
""";
/*
| """
| {
| "name": "Joan Smith",
| "phone": "555-123-4567",
| "address": "1 Maple Drive, Anytown"
| }
| """
*/
record Rectangle(String name, double width, double height) {
double area() {
return width * height;
}
}
Rectangle[] zone = new Rectangle[] {
new Rectangle("Alfa", 17.8, 31.4),
new Rectangle("Bravo", 9.6, 12.4),
new Rectangle("Charlie", 7.1, 11.23),
};
String table = STR."""
Description Width Height Area
\{zone[0].name} \{zone[0].width} \{zone[0].height} \{zone[0].area()}
\{zone[1].name} \{zone[1].width} \{zone[1].height} \{zone[1].area()}
\{zone[2].name} \{zone[2].width} \{zone[2].height} \{zone[2].area()}
Total \{zone[0].area() + zone[1].area() + zone[2].area()}
""";
/*
| """
| Description Width Height Area
| Alfa 17.8 31.4 558.92
| Bravo 9.6 12.4 119.03999999999999
| Charlie 7.1 11.23 79.733
| Total 757.693
| """
*/
}
static void FMTTemplateProcessor() {
record Rectangle(String name, double width, double height) {
double area() {
return width * height;
}
};
Rectangle[] zone = new Rectangle[] {
new Rectangle("Alfa", 17.8, 31.4),
new Rectangle("Bravo", 9.6, 12.4),
new Rectangle("Charlie", 7.1, 11.23),
};
String table = FMT."""
Description Width Height Area
%-12s\{zone[0].name} %7.2f\{zone[0].width} %7.2f\{zone[0].height} %7.2f\{zone[0].area()}
%-12s\{zone[1].name} %7.2f\{zone[1].width} %7.2f\{zone[1].height} %7.2f\{zone[1].area()}
%-12s\{zone[2].name} %7.2f\{zone[2].width} %7.2f\{zone[2].height} %7.2f\{zone[2].area()}
\{" ".repeat(28)} Total %7.2f\{zone[0].area() + zone[1].area() + zone[2].area()}
""";
/*
| """
| Description Width Height Area
| Alfa 17.80 31.40 558.92
| Bravo 9.60 12.40 119.04
| Charlie 7.10 11.23 79.73
| Total 757.69
| """
*/
}
static void ensuringSafety() {
String name = "Joan";
StringTemplate st = RAW."My name is \{name}";
String info = STR.process(st);
}
record User(String firstName, int accountNumber) {}
static void literalsInsideTemplateExpressions() {
String s1 = STR."Welcome to your account";
// | "Welcome to your account"
User user = new User("Lisa", 12345);
String s2 = STR."Welcome, \{user.firstName()}, to your account \{user.accountNumber()}";
// | "Welcome, Lisa, to your account 12345"
}
static void emptyEmbeddedExpression() {
String s1=STR."Test \{ }";
}
}