Merge branch 'pr-1819'

docs/pages/release_notes.md

@@ -31,6 +31,14 @@ CPD also understands now double quoted strings, which are supported since versio
 str = "This is a string"
 ```
 
+#### Enhanced C++ support
+
+CPD now supports digit separators in C++ (language module "cpp"). This is a C++14 feature.
+
+Example: `auto integer_literal = 1'000'000;`
+
+The single quotes can be used to add some structure to large numbers.
+
 ### Fixed Issues
 
 *   dart
@@ -53,6 +61,7 @@ str = "This is a string"
 *   [#1803](https://github.com/pmd/pmd/pull/1803): \[dart] \[cpd] Dart escape sequences - [Maikel Steneker](https://github.com/maikelsteneker)
 *   [#1807](https://github.com/pmd/pmd/pull/1807): \[ci] Fix missing local branch issues when executing pmd-regression-tester - [BBG](https://github.com/djydewang)
 *   [#1813](https://github.com/pmd/pmd/pull/1813): \[matlab] \[cpd] Matlab comments - [Maikel Steneker](https://github.com/maikelsteneker)
+*   [#1819](https://github.com/pmd/pmd/pull/1819): \[cpp] \[cpd] Add support for digit separators - [Maikel Steneker](https://github.com/maikelsteneker)
 *   [#1821](https://github.com/pmd/pmd/pull/1821): \[matlab] \[cpd] Matlab question mark token - [Maikel Steneker](https://github.com/maikelsteneker)
 *   [#1822](https://github.com/pmd/pmd/pull/1822): \[matlab] \[cpd] Double quoted string - [Maikel Steneker](https://github.com/maikelsteneker)
 

pmd-cpp/etc/grammar/cpp.jj

@@ -284,27 +284,31 @@ TOKEN :
 
 TOKEN [IGNORE_CASE] :
 {
-  <  OCTALINT : "0" (["0"-"7"])* >
+  <  OCTALINT : "0" (["'", "0"-"7"])* >
 | <  OCTALLONG : <OCTALINT> "l" >
 | <  UNSIGNED_OCTALINT : <OCTALINT> "u" >
 | <  UNSIGNED_OCTALLONG : <OCTALINT> ("ul" | "lu") >
 
-| <  DECIMALINT : ["1"-"9"] (["0"-"9"])* >
+| < #DECIMALDIGIT : ["'", "0"-"9"] >
+
+| <  DECIMALINT : ["1"-"9"] (<DECIMALDIGIT>)* >
 | <  DECIMALLONG : <DECIMALINT> ["u","l"] >
 | <  UNSIGNED_DECIMALINT : <DECIMALINT> "u" >
 | <  UNSIGNED_DECIMALLONG : <DECIMALINT> ("ul" | "lu") >
 
 
-| <  HEXADECIMALINT : "0x" (["0"-"9","a"-"f"])+ >
+| <  HEXADECIMALINT : "0x" (<DECIMALDIGIT> | ["a"-"f"])+ >
 | <  HEXADECIMALLONG : <HEXADECIMALINT> (["u","l"])? >
 | <  UNSIGNED_HEXADECIMALINT : <HEXADECIMALINT> "u" >
 | <  UNSIGNED_HEXADECIMALLONG : <HEXADECIMALINT> ("ul" | "lu") >
 
 
-| <  FLOATONE : ((["0"-"9"])+ "." (["0"-"9"])* | (["0"-"9"])* "." (["0"-"9"])+)
-              ("e" (["-","+"])? (["0"-"9"])+)? (["f","l"])? >
+| <  FLOATONE : (["0"-"9"](<DECIMALDIGIT>)* "."
+              | "." (<DECIMALDIGIT>)+
+              | ["0"-"9"](<DECIMALDIGIT>)* "." (<DECIMALDIGIT>)+)
+              ("e" (["-","+"])? (<DECIMALDIGIT>)+)? (["f","l"])? >
 
-| <  FLOATTWO : (["0"-"9"])+ "e" (["-","+"])?  (["0"-"9"])+ (["f","l"])? >
+| <  FLOATTWO : ["0"-"9"](<DECIMALDIGIT>)* "e" (["-","+"])? (<DECIMALDIGIT>)+ (["f","l"])? >
 }
 
 TOKEN :

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java

@@ -159,6 +159,18 @@ public class CPPTokenizerTest {
         tokenizer.tokenize(code, new Tokens());
     }
 
+    @Test
+    public void testDigitSeparators() {
+        final String code = "auto integer_literal = 1'000'000;" + PMD.EOL
+                + "auto floating_point_literal = 0.000'015'3;" + PMD.EOL
+                + "auto hex_literal = 0x0F00'abcd'6f3d;" + PMD.EOL
+                + "auto silly_example = 1'0'0'000'00;";
+        Tokens tokens = parse(code);
+        assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
+        assertEquals("1'000'000", tokens.getTokens().get(3).toString());
+        assertEquals(21, tokens.size());
+    }
+
     private Tokens parse(String snippet) {
         try {
             return parse(snippet, false, new Tokens());