From d16751d136fe4c6e4baf2a8c91b4abe69c3b5298 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 27 Feb 2020 21:11:02 +0100 Subject: [PATCH] [java] Add support for TextBlocks in Java14 * New escape sequence "\s" added * Added experimental ASTLiteral::getTextBlockContent to retrieve the text block with stripped indentation --- pmd-java/etc/grammar/Java.jjt | 18 ++- .../pmd/lang/java/ast/ASTLiteral.java | 102 ++++++++++++++- .../pmd/lang/java/ast/Java14PreviewTest.java | 119 ++++++++++++++++++ .../pmd/lang/java/ast/Java14Test.java | 3 + .../jdkversiontests/java14/TextBlocks.java | 100 +++++++++++++++ 5 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14PreviewTest.java create mode 100644 pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/ast/jdkversiontests/java14/TextBlocks.java diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index 289bf146ce..fa6df16f37 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -453,8 +453,14 @@ public class JavaParser { } private void checkForTextBlockLiteral() { - if (jdkVersion != 13 || !preview) { - throwParseException("Text block literals are only supported with Java 13 Preview"); + if (jdkVersion != 13 && jdkVersion != 14 || !preview) { + throwParseException("Text block literals are only supported with Java 13 Preview or Java 14 Preview"); + } + } + + private void checkForNewStringSpaceEscape(String s) { + if ((jdkVersion != 14 || !preview) && s.contains("\\s")) { + throwParseException("The escape sequence \"\\s\" is only supported with Java 14 Preview"); } } @@ -688,7 +694,7 @@ TOKEN : > | < #STRING_ESCAPE: "\\" - ( ["n","t","b","r","f","\\","'","\""] + ( ["n","t","b","r","f", "s", "\\","'","\""] // octal escapes | ["0"-"7"] ( ["0"-"7"] )? | ["0"-"3"] ["0"-"7"] ["0"-"7"] @@ -705,7 +711,7 @@ TOKEN : | < TEXT_BLOCK_LITERAL: "\"\"\"" ()* - ( ~["\"", "\\"] | "\"" ~["\""] | "\"\"" ~["\""] | )* + ( ~["\"", "\\"] | "\"" ~["\""] | "\"\"" ~["\""] | | "\\" )* "\"\"\"" > } @@ -1636,8 +1642,8 @@ void Literal() : | t= { checkForBadNumericalLiteralslUsage(t); jjtThis.setImage(t.image); jjtThis.setFloatLiteral();} | t= { checkForBadHexFloatingPointLiteral(); checkForBadNumericalLiteralslUsage(t); jjtThis.setImage(t.image); jjtThis.setFloatLiteral();} | t= {jjtThis.setImage(t.image); jjtThis.setCharLiteral();} -| t= {jjtThis.setImage(t.image); jjtThis.setStringLiteral();} -| t= { checkForTextBlockLiteral(); jjtThis.setImage(t.image); jjtThis.setStringLiteral();} +| t= {jjtThis.setImage(t.image); checkForNewStringSpaceEscape(t.image); jjtThis.setStringLiteral(); } +| t= { checkForTextBlockLiteral(); checkForNewStringSpaceEscape(t.image); jjtThis.setImage(t.image); jjtThis.setStringLiteral();} | BooleanLiteral() | NullLiteral() } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTLiteral.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTLiteral.java index 7bb2a3e4f9..1920e42b69 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTLiteral.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTLiteral.java @@ -5,13 +5,19 @@ package net.sourceforge.pmd.lang.java.ast; import java.math.BigInteger; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; + +import net.sourceforge.pmd.annotation.Experimental; import net.sourceforge.pmd.annotation.InternalApi; public class ASTLiteral extends AbstractJavaTypeNode { + private static final String TEXTBLOCK_DELIMITER = "\"\"\""; private boolean isInt; private boolean isFloat; private boolean isChar; @@ -252,6 +258,100 @@ public class ASTLiteral extends AbstractJavaTypeNode { } public boolean isTextBlock() { - return isString && getImage().startsWith("\"\"\""); + return isString && getImage().startsWith(TEXTBLOCK_DELIMITER); + } + + /** + * Returns the content of the text block after normalizing line endings to LF, + * removing incidental white space surrounding the text block and interpreting + * escape sequences. + */ + @Experimental + public String getTextBlockContent() { + if (!isTextBlock()) { + throw new IllegalArgumentException("This is not a text block"); + } + + int start = determineContentStart(getImage()); + String content = getImage().substring(start, getImage().length() - TEXTBLOCK_DELIMITER.length()); + // normalize line endings to LF + content = content.replaceAll("\r\n|\r", "\n"); + + int prefixLength = Integer.MAX_VALUE; + List lines = Arrays.asList(content.split("\\n")); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + // compute common prefix + if (!StringUtils.isAllBlank(line) || i == lines.size() - 1) { + prefixLength = Math.min(prefixLength, countLeadingWhitespace(line)); + } + } + if (prefixLength == Integer.MAX_VALUE) { + // common prefix not found + prefixLength = 0; + } + StringBuilder sb = new StringBuilder(content.length()); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + // remove common whitespace prefix + if (!StringUtils.isAllBlank(line) && line.length() >= prefixLength) { + line = line.substring(prefixLength); + } + line = removeTrailingWhitespace(line); + sb.append(line); + + boolean isLastLine = i == lines.size() - 1; + boolean isFirstLine = i == 0; + if (!isLastLine || !isFirstLine && !StringUtils.isAllBlank(line)) { + sb.append('\n'); + } + } + String result = sb.toString(); + + // interpret escape sequences "\NL" "n","t","b","r","f", "s", "\"", "\'" + result = result + .replaceAll("\\\\\n", "") + .replaceAll("\\\\n", "\n") + .replaceAll("\\\\t", "\t") + .replaceAll("\\\\b", "\b") + .replaceAll("\\\\r", "\r") + .replaceAll("\\\\f", "\f") + .replaceAll("\\\\s", " ") + .replaceAll("\\\\\"", "\"") + .replaceAll("\\\\'", "'"); + return result; + } + + private static int determineContentStart(String s) { + int start = TEXTBLOCK_DELIMITER.length(); // this is the opening delimiter + boolean lineTerminator = false; + // the content begins after at the first character after the line terminator + // of the opening delimiter + while (start < s.length() && Character.isWhitespace(s.charAt(start))) { + if (s.charAt(start) == '\r' || s.charAt(start) == '\n') { + lineTerminator = true; + } else if (lineTerminator) { + // first character after the line terminator + break; + } + start++; + } + return start; + } + + private static int countLeadingWhitespace(String s) { + int count = 0; + while (count < s.length() && Character.isWhitespace(s.charAt(count))) { + count++; + } + return count; + } + + private static String removeTrailingWhitespace(String s) { + int endIndexIncluding = s.length() - 1; + while (endIndexIncluding >= 0 && Character.isWhitespace(s.charAt(endIndexIncluding))) { + endIndexIncluding--; + } + return s.substring(0, endIndexIncluding + 1); } } diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14PreviewTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14PreviewTest.java new file mode 100644 index 0000000000..30b1d7daa9 --- /dev/null +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14PreviewTest.java @@ -0,0 +1,119 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + + +package net.sourceforge.pmd.lang.java.ast; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import net.sourceforge.pmd.lang.ast.ParseException; +import net.sourceforge.pmd.lang.java.JavaParsingHelper; + +/** + * Tests new java14 preview features. + */ +public class Java14PreviewTest { + private final JavaParsingHelper java14 = + JavaParsingHelper.WITH_PROCESSING.withDefaultVersion("14") + .withResourceContext(Java14Test.class, "jdkversiontests/java14/"); + + private final JavaParsingHelper java14p = java14.withDefaultVersion("14-preview"); + private final JavaParsingHelper java13 = java14.withDefaultVersion("13"); + + @Test + public void textBlocks() { + ASTCompilationUnit compilationUnit = java14p.parseResource("TextBlocks.java"); + List literals = compilationUnit.findDescendantsOfType(ASTLiteral.class); + Assert.assertEquals(16, literals.size()); + Assert.assertFalse(literals.get(2).isTextBlock()); + Assert.assertFalse(literals.get(12).isTextBlock()); + + List textBlocks = new ArrayList<>(); + for (ASTLiteral literal : literals) { + if (literal.isTextBlock()) { + textBlocks.add(literal); + } + } + Assert.assertEquals(14, textBlocks.size()); + Assert.assertEquals("\"\"\"\n" + + " \n" + + " \n" + + "

Hello, world

\n" + + " \n" + + " \n" + + " \"\"\"", + textBlocks.get(0).getImage()); + Assert.assertEquals("\n" + + " \n" + + "

Hello, world

\n" + + " \n" + + "\n", textBlocks.get(0).getTextBlockContent()); + + // with escapes + Assert.assertEquals("\"\"\"\n" + + " \\r\n" + + " \\r\n" + + "

Hello, world

\\r\n" + + " \\r\n" + + " \\r\n" + + " \"\"\"", textBlocks.get(3).getImage()); + Assert.assertEquals("\r\n" + + " \r\n" + + "

Hello, world

\r\n" + + " \r\n" + + "\r\n", textBlocks.get(3).getTextBlockContent()); + // season + Assert.assertEquals("\"\"\"\n winter\"\"\"", textBlocks.get(4).getImage()); + Assert.assertEquals("winter", textBlocks.get(4).getTextBlockContent()); + // period + Assert.assertEquals("\"\"\"\n" + + " winter\n" + + " \"\"\"", textBlocks.get(5).getImage()); + Assert.assertEquals("winter\n", textBlocks.get(5).getTextBlockContent()); + // empty + Assert.assertEquals("\"\"\"\n \"\"\"", textBlocks.get(8).getImage()); + Assert.assertEquals("", textBlocks.get(8).getTextBlockContent()); + // escaped text block in inside text block + Assert.assertEquals("\"\"\"\n" + + " String text = \\\"\"\"\n" + + " A text block inside a text block\n" + + " \\\"\"\";\n" + + " \"\"\"", textBlocks.get(11).getImage()); + Assert.assertEquals("String text = \"\"\"\n" + + " A text block inside a text block\n" + + "\"\"\";\n", textBlocks.get(11).getTextBlockContent()); + // new escape: line continuation + Assert.assertEquals("\"\"\"\n" + + " Lorem ipsum dolor sit amet, consectetur adipiscing \\\n" + + " elit, sed do eiusmod tempor incididunt ut labore \\\n" + + " et dolore magna aliqua.\\\n" + + " \"\"\"", textBlocks.get(12).getImage()); + Assert.assertEquals("Lorem ipsum dolor sit amet, consectetur adipiscing " + + "elit, sed do eiusmod tempor incididunt ut labore " + + "et dolore magna aliqua.", textBlocks.get(12).getTextBlockContent()); + // new escape: space escape + Assert.assertEquals("\"\"\"\n" + + " red \\s\n" + + " green\\s\n" + + " blue \\s\n" + + " \"\"\"", textBlocks.get(13).getImage()); + Assert.assertEquals("red \n" + + "green \n" + + "blue \n", textBlocks.get(13).getTextBlockContent()); + } + + @Test(expected = ParseException.class) + public void textBlocksBeforeJava14PreviewShouldFail() { + java13.parseResource("TextBlocks.java"); + } + + @Test(expected = ParseException.class) + public void stringEscapeSequenceShouldFail() { + java14.parse("class Foo { String s =\"a\\sb\"; }"); + } +} diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14Test.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14Test.java index b2cb5e5d0c..c05e1736c4 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14Test.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/Java14Test.java @@ -13,6 +13,9 @@ import org.junit.Test; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.java.JavaParsingHelper; +/** + * Tests new java14 standard features. + */ public class Java14Test { private final JavaParsingHelper java14 = JavaParsingHelper.WITH_PROCESSING.withDefaultVersion("14") diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/ast/jdkversiontests/java14/TextBlocks.java b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/ast/jdkversiontests/java14/TextBlocks.java new file mode 100644 index 0000000000..0502961c4c --- /dev/null +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/ast/jdkversiontests/java14/TextBlocks.java @@ -0,0 +1,100 @@ +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; + +/** + * @see JEP 368: Text Blocks (Second Preview) + */ +public class TextBlocks { + + + public static void main(String[] args) throws Exception { + // note: there is trailing whitespace!! + String html = """ + + +

Hello, world

+ + + """; + System.out.println(html); + + String query = """ + SELECT `EMP_ID`, `LAST_NAME` FROM `EMPLOYEE_TB` + WHERE `CITY` = 'INDIANAPOLIS' + ORDER BY `EMP_ID`, `LAST_NAME`; + """; + System.out.println(query); + + ScriptEngine engine = new ScriptEngineManager().getEngineByName("js"); + Object obj = engine.eval(""" + function hello() { + print('"Hello, world"'); + } + + hello(); + """); + + // Escape sequences + String htmlWithEscapes = """ + \r + \r +

Hello, world

\r + \r + \r + """; + System.out.println(htmlWithEscapes); + + String season = """ + winter"""; // the six characters w i n t e r + + String period = """ + winter + """; // the seven characters w i n t e r LF + + String greeting = + """ + Hi, "Bob" + """; // the ten characters H i , SP " B o b " LF + + String salutation = + """ + Hi, + "Bob" + """; // the eleven characters H i , LF SP " B o b " LF + + String empty = """ + """; // the empty string (zero length) + + String quote = """ + " + """; // the two characters " LF + + String backslash = """ + \\ + """; // the two characters \ LF + + String normalStringLiteral = "test"; + + String code = + """ + String text = \""" + A text block inside a text block + \"""; + """; + + // new escape sequences + String text = """ + Lorem ipsum dolor sit amet, consectetur adipiscing \ + elit, sed do eiusmod tempor incididunt ut labore \ + et dolore magna aliqua.\ + """; + System.out.println(text); + + String colors = """ + red \s + green\s + blue \s + """; + System.out.println(colors); + } +}