Call jjtree/javacc with UTF-8 as default encoding

Fixes #2970

pmd-core/src/main/ant/alljavacc.xml

@@ -30,15 +30,23 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/ast/dummy" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/ast/dummy" />
         <echo>Using JavaCC home: ${javacc-home.path}</echo>
-        <jjtree target="etc/grammar/dummy.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/ast/dummy"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/ast/dummy" />
+            <arg value="etc/grammar/dummy.jjt" />
+        </java>
 
         <!-- Generate CharStream interface -->
-        <javacc usercharstream="true"
+        <java fork="true"
-                target="${target}/net/sourceforge/pmd/lang/ast/dummy/dummy.jj"
+              classname="javacc"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/ast/dummy"
+              classpath="${javacc-home.path}/javacc.jar">
-                javacchome="${javacc-home.path}" />
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/ast/dummy" />
+            <arg value="${target}/net/sourceforge/pmd/lang/ast/dummy/dummy.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/ast/dummy/CharStream.java"
                  token="net.sourceforge.pmd.lang.ast.dummy"
                  value="net.sourceforge.pmd.lang.ast" />

pmd-cpp/src/main/ant/alljavacc.xml

@@ -29,11 +29,15 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/cpp/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/cpp/ast" />
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="etc/grammar/cpp.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/cpp/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/cpp/ast" />
+            <arg value="etc/grammar/cpp.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/cpp/ast/CppParserTokenManager.java"
                  token="class CppParserTokenManager"
                  value="class CppParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />

pmd-java/src/main/ant/alljavacc.xml

@@ -28,15 +28,23 @@
     <target name="javajjtree" description="Generates the Java parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/java/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/java/ast" />
-        <jjtree target="etc/grammar/Java.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/java/ast"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/java/ast" />
+            <arg value="etc/grammar/Java.jjt" />
+        </java>
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="${target}/net/sourceforge/pmd/lang/java/ast/Java.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/java/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/java/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/java/ast/Java.jj" />
+        </java>
         <delete file="${target}/net/sourceforge/pmd/lang/java/ast/Node.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/java/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/java/ast/CharStream.java" />

pmd-javascript/src/main/ant/alljavacc.xml

@@ -29,11 +29,15 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/ecmascript5/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/ecmascript5/ast" />
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="etc/grammar/es5.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/ecmascript5/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/ecmascript5/ast" />
+            <arg value="etc/grammar/es5.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/ecmascript5/ast/Ecmascript5ParserTokenManager.java"
                  token="class Ecmascript5ParserTokenManager"
                  value="class Ecmascript5ParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />

pmd-jsp/src/main/ant/alljavacc.xml

@@ -28,17 +28,25 @@
     <target name="jspjjtree" description="Generates the JSP parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/jsp/ast/" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/jsp/ast/" />
-        <jjtree target="etc/grammar/JspParser.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/jsp/ast/"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/jsp/ast" />
+            <arg value="etc/grammar/JspParser.jjt" />
+        </java>
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                unicodeinput="true"
+              classpath="${javacc-home.path}/javacc.jar">
-                javaunicodeescape="false"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                target="${target}/net/sourceforge/pmd/lang/jsp/ast/JspParser.jj"
+            <arg value="-STATIC:false" />
-                outputdirectory="${target}/net/sourceforge/pmd/lang/jsp/ast"
+            <arg value="-USER_CHAR_STREAM:true" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-UNICODE_INPUT:true" />
+            <arg value="-JAVA_UNICODE_ESCAPE:false" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/jsp/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/jsp/ast/JspParser.jj" />
+        </java>
         <delete file="${target}/net/sourceforge/pmd/lang/jsp/ast/Node.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/jsp/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/jsp/ast/CharStream.java" />

pmd-matlab/src/main/ant/alljavacc.xml

@@ -29,11 +29,15 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/matlab/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/matlab/ast" />
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="etc/grammar/matlab.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/matlab/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/matlab/ast" />
+            <arg value="etc/grammar/matlab.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/matlab/ast/MatlabParserTokenManager.java"
                  token="class MatlabParserTokenManager"
                  value="class MatlabParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />

pmd-modelica/src/main/ant/alljavacc.xml

@@ -28,15 +28,23 @@
     <target name="modelicajjtree" description="Generates the Modelica parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/modelica/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/modelica/ast" />
-        <jjtree target="etc/grammar/Modelica.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/modelica/ast"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/modelica/ast" />
+            <arg value="etc/grammar/Modelica.jjt" />
+        </java>
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="${target}/net/sourceforge/pmd/lang/modelica/ast/Modelica.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/modelica/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/modelica/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/modelica/ast/Modelica.jj" />
+        </java>
         <delete file="${target}/net/sourceforge/pmd/lang/modelica/ast/Node.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/modelica/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/modelica/ast/CharStream.java" />

pmd-objectivec/src/main/ant/alljavacc.xml

@@ -29,11 +29,15 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/objectivec/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/objectivec/ast" />
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="etc/grammar/ObjC2.0.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/objectivec/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/objectivec/ast" />
+            <arg value="etc/grammar/ObjC2.0.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/objectivec/ast/ObjectiveCParserTokenManager.java"
                  token="class ObjectiveCParserTokenManager"
                  value="class ObjectiveCParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />

pmd-plsql/src/main/ant/alljavacc.xml

@@ -28,15 +28,23 @@
     <target name="plsqljjtree" description="Generates the PlSQL parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/plsql/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/plsql/ast" />
-        <jjtree target="etc/grammar/PldocAST.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/plsql/ast"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/plsql/ast" />
+            <arg value="etc/grammar/PldocAST.jjt" />
+        </java>
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="false"
+              classname="javacc"
-                target="${target}/net/sourceforge/pmd/lang/plsql/ast/PldocAST.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/plsql/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:false" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/plsql/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/plsql/ast/PldocAST.jj" />
+        </java>
 
         <delete file="${target}/net/sourceforge/pmd/lang/plsql/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/plsql/ast/Node.java" />

pmd-python/src/main/ant/alljavacc.xml

@@ -29,11 +29,15 @@
         <delete dir="${target}/net/sourceforge/pmd/lang/python/ast" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/python/ast" />
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                target="etc/grammar/python.jj"
+              classpath="${javacc-home.path}/javacc.jar">
-                outputdirectory="${target}/net/sourceforge/pmd/lang/python/ast"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/python/ast" />
+            <arg value="etc/grammar/python.jj" />
+        </java>
         <replace file="${target}/net/sourceforge/pmd/lang/python/ast/PythonParserTokenManager.java"
                  token="class PythonParserTokenManager"
                  value="class PythonParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />

pmd-visualforce/src/main/ant/alljavacc.xml

@@ -28,18 +28,27 @@
     <target name="jspjjtree" description="Generates the VF parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/vf/ast/" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/vf/ast/" />
-        <jjtree target="etc/grammar/VfParser.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/vf/ast/"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
-        <!-- Ensure generated using CharStream interface debugparser="true"
+            <sysproperty key="file.encoding" value="UTF-8" />
-        		debugtokenmanager="true"-->
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/vf/ast" />
-        <javacc static="false"
+            <arg value="etc/grammar/VfParser.jjt" />
-                usercharstream="true"
+        </java>
-                unicodeinput="true"
+        <!-- Ensure generated using CharStream interface -->
-                javaunicodeescape="false"
+        <java fork="true"
-                target="${target}/net/sourceforge/pmd/lang/vf/ast/VfParser.jj"
+              classname="javacc"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/vf/ast"
+              classpath="${javacc-home.path}/javacc.jar">
-                javacchome="${javacc-home.path}" />
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-STATIC:false" />
+            <arg value="-USER_CHAR_STREAM:true" />
+            <arg value="-UNICODE_INPUT:true" />
+            <arg value="-JAVA_UNICODE_ESCAPE:false" />
+            <!-- <arg value="-DEBUG_PARSER:true" /> -->
+            <!-- <arg value="-DEBUG_TOKEN_MANAGER:true" /> -->
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/vf/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/vf/ast/VfParser.jj" />
+        </java>
         <delete file="${target}/net/sourceforge/pmd/lang/vf/ast/Node.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/vf/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/vf/ast/CharStream.java" />

pmd-vm/src/main/ant/alljavacc.xml

@@ -28,17 +28,25 @@
     <target name="vmjjtree" description="Generates the VM parser and AST source files" unless="javaccBuildNotRequired">
         <delete dir="${target}/net/sourceforge/pmd/lang/vm/ast/" />
         <mkdir dir="${target}/net/sourceforge/pmd/lang/vm/ast/" />
-        <jjtree target="etc/grammar/VmParser.jjt"
+        <java fork="true"
-                outputdirectory="${target}/net/sourceforge/pmd/lang/vm/ast/"
+              classname="jjtree"
-                javacchome="${javacc-home.path}" />
+              classpath="${javacc-home.path}/javacc.jar">
+            <sysproperty key="file.encoding" value="UTF-8" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/vm/ast" />
+            <arg value="etc/grammar/VmParser.jjt" />
+        </java>
         <!-- Ensure generated using CharStream interface -->
-        <javacc static="false"
+        <java fork="true"
-                usercharstream="true"
+              classname="javacc"
-                unicodeinput="true"
+              classpath="${javacc-home.path}/javacc.jar">
-                javaunicodeescape="false"
+            <sysproperty key="file.encoding" value="UTF-8" />
-                target="${target}/net/sourceforge/pmd/lang/vm/ast/VmParser.jj"
+            <arg value="-STATIC:false" />
-                outputdirectory="${target}/net/sourceforge/pmd/lang/vm/ast"
+            <arg value="-USER_CHAR_STREAM:true" />
-                javacchome="${javacc-home.path}" />
+            <arg value="-UNICODE_INPUT:true" />
+            <arg value="-JAVA_UNICODE_ESCAPE:false" />
+            <arg value="-OUTPUT_DIRECTORY:${target}/net/sourceforge/pmd/lang/vm/ast" />
+            <arg value="${target}/net/sourceforge/pmd/lang/vm/ast/VmParser.jj" />
+        </java>
         <delete file="${target}/net/sourceforge/pmd/lang/vm/ast/Node.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/vm/ast/SimpleNode.java" />
         <delete file="${target}/net/sourceforge/pmd/lang/vm/ast/CharStream.java" />