diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index 520e435232..5e506a140e 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -521,67 +521,53 @@ TOKEN : TOKEN : { < INTEGER_LITERAL: - (["l","L"])? - | (["l","L"])? - | (["l","L"])? - | (["l","L"])? + (["l","L"])? + | (["l","L"])? + | (["l","L"])? + | (["l","L"])? > -| - < #DECIMAL_LITERAL: (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) > -| - < #HEX_LITERAL: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"]((["0"-"9","a"-"f","A"-"F","_"])*["0"-"9","a"-"f","A"-"F"])?) > -| - < #BINARY_LITERAL: "0" ["b","B"] (["0","1"]((["0","1","_"])*["0","1"])?) > -| - < #OCTAL_LITERAL: "0" (["0"-"7"]((["0"-"7","_"])*["0"-"7"])?) > +| < #DECIMAL_NUMERAL: ["1"-"9"] (("_")* ["0"-"9"])* > +| < #HEX_NUMERAL: "0" ["x","X"] > +| < #BINARY_NUMERAL: "0" ["b","B"] ["0","1"] (("_")* ["0","1"])* > +| < #OCTAL_NUMERAL: "0" (("_")* ["0"-"7"])* > | < FLOATING_POINT_LITERAL: - (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) "." (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?)? ()? (["f","F","d","D"])? - | "." (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) ()? (["f","F","d","D"])? - | (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) (["f","F","d","D"])? - | (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) ()? ["f","F","d","D"] + + | > -| - < HEX_FLOATING_POINT_LITERAL: - ( (".")? | "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"]((["0"-"9","a"-"f","A"-"F","_"])*["0"-"9","a"-"f","A"-"F"])?)? "." (["0"-"9","a"-"f","A"-"F"]((["0"-"9","a"-"f","A"-"F","_"])*["0"-"9","a"-"f","A"-"F"])?)) ["p","P"] (["+","-"])? (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) (["f","F","d","D"])? +| < #DECIMAL_FLOATING_POINT_LITERAL: + "." ()? ()? (["f","F", "d","D"])? + | "." ()? (["f","F", "d","D"])? + | (["f","F", "d","D"])? + | ()? ["f","F", "d","D"] > -| - < #EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"]((["0"-"9","_"])*["0"-"9"])?) > -| - < CHARACTER_LITERAL: - "'" - ( (~["'","\\","\n","\r"]) - | ("\\" - ( ["n","t","b","r","f","\\","'","\""] - | ["0"-"7"] ( ["0"-"7"] )? - | ["0"-"3"] ["0"-"7"] ["0"-"7"] - ) - ) - ) - "'" - > -| < #STRING_ESCAPE: - "\\" - ( ["n","t","b","r","f","\\","'","\""] - // octal escapes - | ["0"-"7"] ( ["0"-"7"] )? - | ["0"-"3"] ["0"-"7"] ["0"-"7"] - ) - > -| - < STRING_LITERAL: - "\"" - ( (~["\"","\\","\n","\r"]) - | - )* - "\"" +| < #HEX_FLOATING_POINT_LITERAL: + "0" ["x","X"] (".")? (["f","F", "d","D"])? + | "0" ["x","X"] ()? "." (["f","F", "d","D"])? > +| < #DIGIT_SEQ: ["0"-"9"] (("_")* ["0"-"9"])* > +| < #HEX_DIGIT_SEQ: ["0"-"9","a"-"f","A"-"F"] (("_")* ["0"-"9","a"-"f","A"-"F"])* > + +| < #EXPONENT: ["e","E"] > +| < #HEX_EXPONENT: ["p","P"] > +| < #EXPONENT_TAIL: (["+","-"])? > + +| < CHARACTER_LITERAL: "'" ( ~["'", "\\","\n","\r"] | ) "'" > +| < STRING_LITERAL: "\"" ( ~["\"","\\","\n","\r"] | )* "\"" > | < TEXT_BLOCK_LITERAL: "\"\"\"" ()* ( ~["\"", "\\"] | "\"" ~["\""] | "\"\"" ~["\""] | )* "\"\"\"" > +| < #STRING_ESCAPE: + "\\" + ( ["n","t","b","r","f","\\","'","\""] + // octal escapes + | ["0"-"7"] ( ["0"-"7"] )? + | ["0"-"3"] ["0"-"7"] ["0"-"7"] + ) + > } /* IDENTIFIERS */ @@ -1829,7 +1815,6 @@ void NumericLiteral(): { ( t= { jjtThis.setIntLiteral(); } | t= { jjtThis.setFloatLiteral(); } - | t= { jjtThis.setFloatLiteral(); } ) { jjtThis.setImage(t.image); } } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 7361a5d4cc..5f1c2f5db0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -62,7 +62,7 @@ public class JavaTokenizer extends JavaCCTokenizer { if (ignoreLiterals && (javaToken.kind == JavaTokenKinds.STRING_LITERAL || javaToken.kind == JavaTokenKinds.CHARACTER_LITERAL - || javaToken.kind == JavaTokenKinds.DECIMAL_LITERAL + || javaToken.kind == JavaTokenKinds.INTEGER_LITERAL || javaToken.kind == JavaTokenKinds.FLOATING_POINT_LITERAL)) { image = String.valueOf(javaToken.kind); } diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt index a4df46b8ec..32dd65d3d0 100644 --- a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt @@ -331,8 +331,11 @@ $delim // this starts with zero so is octal, // but 9 is too big for an octal digit - // "099" shouldNot parse() - // "099" shouldNot parse() + "099" shouldNot parse() + "00_8" shouldNot parse() + "08_" shouldNot parse() + "0x8_" shouldNot parse() + "8_" shouldNot parse() "0b" shouldNot parse() "0x" shouldNot parse() "0" should parseAs {