diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml
index 74e4cc4ecb..1cc67fb2c9 100644
--- a/pmd-core/src/main/ant/alljavacc.xml
+++ b/pmd-core/src/main/ant/alljavacc.xml
@@ -2,6 +2,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -27,88 +38,98 @@
-
-
+
+
Using JavaCC home: ${javacc-home.path}
-
-
-
-
+
+
+
+
+
+
+
+
-
-
+
-
+ file="${tmp-package.dir}/JavaCharStream.java"
+ tofile="${target-package.dir}/JavaCharStreamBase.java" />
-
-
-
-
-
+
+
-
-
-
-
-
-
-
+
+
+ "Lexical error in file " + net.sourceforge.pmd.lang.ast.impl.javacc.AbstractTokenManager.getFileName() + " at line "
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pmd-cpp/etc/grammar/cpp.jj b/pmd-cpp/etc/grammar/cpp.jj
index 6367635f4d..ed0145dda2 100644
--- a/pmd-cpp/etc/grammar/cpp.jj
+++ b/pmd-cpp/etc/grammar/cpp.jj
@@ -32,7 +32,7 @@ options {
PARSER_BEGIN(CppParser)
package net.sourceforge.pmd.lang.cpp.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public final class CppParser {
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java
index 97e9540b85..8afb53f0d9 100644
--- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java
@@ -8,7 +8,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.regex.Pattern;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
/**
* A SimpleCharStream, that supports the continuation of lines via backslash+newline,
diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt
index 098c0369dd..c938b8aed5 100644
--- a/pmd-java/etc/grammar/Java.jjt
+++ b/pmd-java/etc/grammar/Java.jjt
@@ -250,7 +250,7 @@ package net.sourceforge.pmd.lang.java.ast;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java
index e817d8e60b..6272069f78 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenFactory.java
@@ -4,7 +4,7 @@
package net.sourceforge.pmd.lang.java.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.TokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaCharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
diff --git a/pmd-javascript/etc/grammar/es5.jj b/pmd-javascript/etc/grammar/es5.jj
index 688692d61d..4878bc295f 100644
--- a/pmd-javascript/etc/grammar/es5.jj
+++ b/pmd-javascript/etc/grammar/es5.jj
@@ -15,7 +15,7 @@ options {
PARSER_BEGIN(Ecmascript5Parser)
package net.sourceforge.pmd.lang.ecmascript5.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class Ecmascript5Parser {
diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript5/Ecmascript5TokenManager.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript5/Ecmascript5TokenManager.java
index bdfea305bc..292801dd6f 100644
--- a/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript5/Ecmascript5TokenManager.java
+++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript5/Ecmascript5TokenManager.java
@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.ecmascript5;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserTokenManager;
/**
diff --git a/pmd-jsp/etc/grammar/JspParser.jjt b/pmd-jsp/etc/grammar/JspParser.jjt
index 08b9218a32..af2c3540e7 100644
--- a/pmd-jsp/etc/grammar/JspParser.jjt
+++ b/pmd-jsp/etc/grammar/JspParser.jjt
@@ -15,26 +15,26 @@
* Script code inside is not parsed.
*/
-options {
+options {
USER_CHAR_STREAM = true;
NODE_USES_PARSER=true;
UNICODE_INPUT=true;
- FORCE_LA_CHECK = false;
- IGNORE_CASE = true;
- STATIC = false;
-
+ FORCE_LA_CHECK = false;
+ IGNORE_CASE = true;
+ STATIC = false;
+
MULTI=true;
VISITOR=true;
TRACK_TOKENS = true;
-}
+}
PARSER_BEGIN(JspParser)
package net.sourceforge.pmd.lang.jsp.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
-/**
+/**
* JSP Parser for PMD.
* @author Pieter, Application Engineers NV/SA, http://www.ae.be
*/
diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/JspParser.java
index e311281975..84a0990805 100644
--- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/JspParser.java
+++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/JspParser.java
@@ -11,7 +11,7 @@ import net.sourceforge.pmd.lang.ParserOptions;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.AbstractTokenManager;
/**
diff --git a/pmd-matlab/etc/grammar/matlab.jj b/pmd-matlab/etc/grammar/matlab.jj
index 31aca621e7..2ab9d58fe0 100644
--- a/pmd-matlab/etc/grammar/matlab.jj
+++ b/pmd-matlab/etc/grammar/matlab.jj
@@ -21,7 +21,7 @@ options {
PARSER_BEGIN(MatlabParser)
package net.sourceforge.pmd.lang.matlab.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class MatlabParser {
@@ -159,4 +159,4 @@ PARSER_END(MatlabParser)
TOKEN :
{
< TR : "'" > : TRANSPOSE
-}
\ No newline at end of file
+}
diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabTokenManager.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabTokenManager.java
index 5b620f7664..4ce5151c09 100644
--- a/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabTokenManager.java
+++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabTokenManager.java
@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.matlab;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.matlab.ast.MatlabParserTokenManager;
/**
diff --git a/pmd-modelica/etc/grammar/Modelica.jjt b/pmd-modelica/etc/grammar/Modelica.jjt
index 33a786817d..494156de46 100644
--- a/pmd-modelica/etc/grammar/Modelica.jjt
+++ b/pmd-modelica/etc/grammar/Modelica.jjt
@@ -48,7 +48,7 @@ options {
PARSER_BEGIN(ModelicaParser)
package net.sourceforge.pmd.lang.modelica.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class ModelicaParser {
diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaParser.java
index f82d5252c9..c0afd23d0b 100644
--- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaParser.java
+++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaParser.java
@@ -11,8 +11,8 @@ import net.sourceforge.pmd.lang.ParserOptions;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.ParseException;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.AbstractTokenManager;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
public class ModelicaParser extends AbstractParser {
diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaTokenManager.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaTokenManager.java
index 9d1e0e6295..ff1d17ea84 100644
--- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaTokenManager.java
+++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ModelicaTokenManager.java
@@ -7,8 +7,8 @@ package net.sourceforge.pmd.lang.modelica;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.AbstractTokenManager;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.modelica.ast.ModelicaParserTokenManager;
diff --git a/pmd-objectivec/etc/grammar/ObjC2.0.jj b/pmd-objectivec/etc/grammar/ObjC2.0.jj
index f25936bdfa..37837f282b 100644
--- a/pmd-objectivec/etc/grammar/ObjC2.0.jj
+++ b/pmd-objectivec/etc/grammar/ObjC2.0.jj
@@ -21,7 +21,7 @@ package net.sourceforge.pmd.lang.objectivec.ast;
import java.io.*;
import java.util.*;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCTokenManager.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCTokenManager.java
index ec573e5d96..0d95984498 100644
--- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCTokenManager.java
+++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/lang/objectivec/ObjectiveCTokenManager.java
@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.objectivec;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.objectivec.ast.ObjectiveCParserTokenManager;
/**
diff --git a/pmd-plsql/etc/grammar/PldocAST.jjt b/pmd-plsql/etc/grammar/PldocAST.jjt
index c164b120a3..08681e1407 100644
--- a/pmd-plsql/etc/grammar/PldocAST.jjt
+++ b/pmd-plsql/etc/grammar/PldocAST.jjt
@@ -155,7 +155,7 @@ package net.sourceforge.pmd.lang.plsql.ast;
import java.io.*;
import net.sourceforge.pmd.lang.ast.Node;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PLSQLParser {
diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/PLSQLTokenManager.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/PLSQLTokenManager.java
index e07ee37d39..48fcc03fbf 100644
--- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/PLSQLTokenManager.java
+++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/PLSQLTokenManager.java
@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.plsql;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserTokenManager;
/**
diff --git a/pmd-python/etc/grammar/python.jj b/pmd-python/etc/grammar/python.jj
index 1f97f9d948..a7ae02074c 100644
--- a/pmd-python/etc/grammar/python.jj
+++ b/pmd-python/etc/grammar/python.jj
@@ -1,23 +1,23 @@
/**
- * This Python 2.7 grammar was copied from the PyDev Project. (http://www.pydev.org/)
- *
- * Original source file:
+ * This Python 2.7 grammar was copied from the PyDev Project. (http://www.pydev.org/)
+ *
+ * Original source file:
* https://github.com/aptana/Pydev/blob/development/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar27/python.jjt (commit 32950d534139f286e03d34795aec99edab09c04c)
*/
-
+
options {
BUILD_PARSER=false;
CACHE_TOKENS=true;
STATIC=false;
UNICODE_INPUT = true;
- USER_CHAR_STREAM=true;
+ USER_CHAR_STREAM=true;
}
PARSER_BEGIN(PythonParser)
package net.sourceforge.pmd.lang.python.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PythonParser {
diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java b/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java
index 30a3aa72fc..c80b6be501 100644
--- a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java
+++ b/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java
@@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.python;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.python.ast.PythonParserTokenManager;
/**
diff --git a/pmd-visualforce/etc/grammar/VfParser.jjt b/pmd-visualforce/etc/grammar/VfParser.jjt
index 5e1bbf20d1..f1ae19b8f3 100644
--- a/pmd-visualforce/etc/grammar/VfParser.jjt
+++ b/pmd-visualforce/etc/grammar/VfParser.jjt
@@ -14,7 +14,7 @@ options {
PARSER_BEGIN(VfParser)
package net.sourceforge.pmd.lang.vf.ast;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class VfParser {
diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfSimpleCharStream.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfSimpleCharStream.java
index a1dd050cd0..b3b1bda907 100644
--- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfSimpleCharStream.java
+++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/VfSimpleCharStream.java
@@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vf;
import java.io.Reader;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
/**
* @author sergey.gorbaty
diff --git a/pmd-vm/etc/grammar/VmParser.jjt b/pmd-vm/etc/grammar/VmParser.jjt
index 4fab39c65a..497449a2c8 100644
--- a/pmd-vm/etc/grammar/VmParser.jjt
+++ b/pmd-vm/etc/grammar/VmParser.jjt
@@ -93,7 +93,7 @@ import java.util.List;
import java.util.HashMap;
import java.util.Map;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.vm.directive.Directive;
import net.sourceforge.pmd.lang.vm.util.VelocityCharStream;
import net.sourceforge.pmd.lang.vm.util.DirectiveMapper;
diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/util/VelocityCharStream.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/util/VelocityCharStream.java
index 78a3e91fa3..cd6b536ec6 100644
--- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/util/VelocityCharStream.java
+++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/util/VelocityCharStream.java
@@ -1,7 +1,7 @@
package net.sourceforge.pmd.lang.vm.util;
-import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
/*
* Licensed to the Apache Software Foundation (ASF) under one