hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Merge branch 'pr-2360' into pmd/7.0.x

Browse Source 
[cpp] Use new javacc wrapper script in C++ module
This commit is contained in:
Andreas Dangel
2020-03-19 15:35:48 +01:00
parent 007b05f610 c4ec36b2b6
commit 319769ac30
8 changed files with 55 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
pmd-cpp/etc/grammar/cpp.jj → pmd-cpp/etc/grammar/Cpp.jj
View File

@ -29,107 +29,16 @@ options {
UNICODE_INPUT=true;
}
PARSER_BEGIN(CppParser)
PARSER_BEGIN(CppParserImpl)
package net.sourceforge.pmd.lang.cpp.ast;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public final class CppParser {
public final class CppParserImpl {
private static String vers = "0.1";
private static String id = "C++ Parser";
private static void msg(String s) {
System.out.println(id + " Version " + vers +": " + s);
}
public static void main(String args[]) {
CppParser parser;
java.io.InputStream input;
int ai = 0;
if (ai == (args.length-1)) {
msg("Reading from file " + args[ai] + " . . .");
try {
input = new java.io.FileInputStream(args[ai]);
} catch (java.io.FileNotFoundException e) {
msg("File " + args[0] + " not found.");
return;
}
} else if (ai >= args.length) {
msg("Reading from standard input . . .");
input = System.in;
} else {
msg("Usage: java " + id + " [-d] [inputfile]");
return;
}
try {
parser = new CppParser(input);
parser.translation_unit();
msg("Program parsed successfully.");
} catch (ParseException e) {
msg("Encountered errors during parse.");
}
}
/**
* A symbol table manager object. Currently only types are recorded for
* doing semantic predicates for parsing.
*/
static SymtabManager sym;
/*
* Methods used in semantics predicates.
*/
/**
* Reads a fully qualified name (since it is used during lookahead, we
* cannot use token. We have to explicitly use getToken).
*/
static String GetFullyScopedName() throws ParseException
{
Token t = getToken(1);
if (t.kind != ID && t.kind != SCOPE)
return null;
StringBuilder s = new StringBuilder();
int i;
if (t.kind != SCOPE)
{
s.append(t.getImage());
t = getToken(2);
i = 3;
}
else
i = 2;
while (t.kind == SCOPE)
{
s.append(t.getImage());
s.append((t = getToken(i++)).getImage());
t = getToken(i++);
}
return s.toString();
}
/**
* This method first tries to read a sequence of tokens of the form
* ("::")? <ID> ("::" <ID>)*
* and if it succeeds then asks the symbol table manager if this is
* the name of a constructor.
*/
static boolean IsCtor() throws ParseException
{
return sym.IsCtor(GetFullyScopedName());
}
}
PARSER_END(CppParser)
PARSER_END(CppParserImpl)
SKIP:
{
@ -337,7 +246,7 @@ TOKEN :
// ^
for (;;) {
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return; }
catch(java.io.IOException e) { return matchedToken; }
if (curChar == '(') break;
sb.append(curChar);
}
@ -348,35 +257,33 @@ rstringbody:
// ^
for (;;) {
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return; }
catch(java.io.IOException e) { return matchedToken; }
if (curChar == ')') {
// delim --------------+
// vvv
// Matching R"...(...)..."
// ^^^
for (int i = 0; i < delim.length(); i++) {
// delim --------------+
// vvv
// Matching R"...(...)..."
// ^^^
for (int i = 0; i < delim.length(); i++) {
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return matchedToken; }
if (delim.charAt(i) != curChar) {
input_stream.backup(1);
continue rstringbody;
}
}
// Matching R"...(...)..."
// ^
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return; }
if (delim.charAt(i) != curChar) {
catch(java.io.IOException e) { return matchedToken; }
if (curChar != '"') {
input_stream.backup(1);
continue rstringbody;
}
}
// Matching R"...(...)..."
// ^
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return; }
if (curChar != '"') {
input_stream.backup(1);
continue rstringbody;
}
break;
break;
}
}
// Setting final token image
matchedToken.image = input_stream.GetImage();
matchedToken.endLine = input_stream.getEndLine();
matchedToken.endColumn = input_stream.getEndColumn();
matchedToken = matchedToken.replaceImage(input_stream);
}
}

6
pmd-cpp/pom.xml
View File

@ -32,8 +32,10 @@
<phase>generate-sources</phase>
<configuration>
<target>
<ant antfile="src/main/ant/alljavacc.xml">
<property name="target" value="${project.build.directory}/generated-sources/javacc" />
<ant antfile="${javacc.ant.wrapper}" target="alljavacc">
<property name="no-jjtree" value="true"/> <!-- This is a CPD module -->
<property name="lang-name" value="Cpp" />
<property name="lang-terse-name" value="cpp" />
<property name="javacc.jar" value="${javacc.jar}" />
</ant>
</target>

113
pmd-cpp/src/main/ant/alljavacc.xml
View File

@ -1,113 +0,0 @@
<project name="pmd" default="alljavacc" basedir="../../">
<property name="javacc-home.path" value="target/lib" />
<target name="alljavacc"
description="Generates all JavaCC aspects within PMD"
depends="checkUpToDate,init,cppjavacc,cleanup" />
<target name="checkUpToDate">
<uptodate property="javaccBuildNotRequired" targetfile="${target}/last-generated-timestamp">
<srcfiles dir="etc/grammar" includes="*.jj*"/>
</uptodate>
<echo message="up to date check: javaccBuildNotRequired=${javaccBuildNotRequired}"/>
</target>
<target name="init" unless="javaccBuildNotRequired">
<mkdir dir="${javacc-home.path}" />
<copy file="${javacc.jar}" tofile="${javacc-home.path}/javacc.jar" />
<mkdir dir="${target}"/>
<touch file="${target}/last-generated-timestamp"/>
</target>
<target name="cleanup">
<delete dir="${javacc-home.path}" />
</target>
<target name="cppjavacc" description="Generates the C++ grammar" unless="javaccBuildNotRequired">
<delete dir="${target}/net/sourceforge/pmd/lang/cpp/ast" />
<mkdir dir="${target}/net/sourceforge/pmd/lang/cpp/ast" />
<!-- Ensure generated using CharStream interface -->
<javacc static="false"
usercharstream="true"
target="etc/grammar/cpp.jj"
outputdirectory="${target}/net/sourceforge/pmd/lang/cpp/ast"
javacchome="${javacc-home.path}" />
<replace file="${target}/net/sourceforge/pmd/lang/cpp/ast/CppParserTokenManager.java"
token="class CppParserTokenManager"
value="class CppParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />
<delete file="${target}/net/sourceforge/pmd/lang/cpp/ast/CharStream.java" />
<delete file="${target}/net/sourceforge/pmd/lang/cpp/ast/ParseException.java" />
<delete file="${target}/net/sourceforge/pmd/lang/cpp/ast/TokenMgrError.java" />
<replace file="${target}/net/sourceforge/pmd/lang/cpp/ast/Token.java">
<replacetoken>public class Token implements java.io.Serializable</replacetoken>
<replacevalue><![CDATA[import net.sourceforge.pmd.lang.ast.GenericToken;
public class Token implements GenericToken, java.io.Serializable]]></replacevalue>
</replace>
<!--Add implementation methods of GenericToken-->
<replace file="${target}/net/sourceforge/pmd/lang/cpp/ast/Token.java">
<replacetoken>public Token specialToken;</replacetoken>
<replacevalue><![CDATA[public Token specialToken;
@Override
public GenericToken getNext() {
return next;
}
@Override
public GenericToken getPreviousComment() {
return specialToken;
}
@Override
public String getImage() {
return image;
}
@Override
public int getBeginLine() {
return beginLine;
}
@Override
public int getEndLine() {
return endLine;
}
@Override
public int getBeginColumn() {
return beginColumn;
}
@Override
public int getEndColumn() {
return endColumn;
}
]]></replacevalue>
</replace>
<replaceregexp>
<regexp pattern="class|interface" />
<substitution expression="@Deprecated @net.sourceforge.pmd.annotation.InternalApi \0" />
<fileset dir="${target}/net/sourceforge/pmd/lang/cpp/ast">
<exclude name="AST*.java" />
</fileset>
</replaceregexp>
<replaceregexp>
<regexp pattern="public class ParseException " />
<substitution expression=" /** @deprecated Use superclass {@link net.sourceforge.pmd.lang.ast.ParseException} */
@Deprecated @net.sourceforge.pmd.annotation.InternalApi \0" />
<fileset file="${target}/net/sourceforge/pmd/lang/cpp/ast/ParseException.java"/>
</replaceregexp>
</target>
</project>

2
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -12,7 +12,7 @@ import java.util.Properties;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.cpp.CppTokenManager;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenManager;
import net.sourceforge.pmd.util.IOUtil;
/**

9
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java → pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java
View File

@ -1,8 +1,8 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp;
package net.sourceforge.pmd.lang.cpp.ast;
import java.io.IOException;
import java.io.Reader;
@ -13,7 +13,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.cpp.ast.CppParserConstants;
/**
* A SimpleCharStream, that supports the continuation of lines via backslash+newline,
@ -71,9 +70,7 @@ class CppCharStream extends SimpleCharStream {
JavaccTokenDocument document = new JavaccTokenDocument(source) {
@Override
protected @Nullable String describeKindImpl(int kind) {
return 0 <= kind && kind < CppParserConstants.tokenImage.length
? CppParserConstants.tokenImage[kind]
: null;
return CppTokenKinds.describe(kind);
}
};
return new CppCharStream(document);

19
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppTokenManager.java → pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppTokenManager.java
View File

@ -1,33 +1,28 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp;
package net.sourceforge.pmd.lang.cpp.ast;
import java.io.Reader;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.cpp.ast.CppParserTokenManager;
/**
* C++ Token Manager implementation.
*
* @deprecated This is internal API
*/
@Deprecated
@InternalApi
public class CppTokenManager implements TokenManager {
private final CppParserTokenManager tokenManager;
public final class CppTokenManager implements TokenManager {
private final CppParserImplTokenManager tokenManager;
/**
* Creates a new C++ Token Manager from the given source code.
*
* @param source
* the source code
* @param source the source code
*/
public CppTokenManager(Reader source) {
tokenManager = new CppParserTokenManager(CppCharStream.newCppCharStream(source));
tokenManager = new CppParserImplTokenManager(CppCharStream.newCppCharStream(source));
}
@Override
@ -37,6 +32,6 @@ public class CppTokenManager implements TokenManager {
@Override
public void setFileName(String fileName) {
CppParserTokenManager.setFileName(fileName);
CppParserImplTokenManager.setFileName(fileName);
}
}

28
pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
View File

@ -19,8 +19,8 @@ import org.junit.Test;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.cpd.SourceCode.StringCodeLoader;
import net.sourceforge.pmd.lang.cpp.CppTokenManager;
import net.sourceforge.pmd.lang.cpp.ast.Token;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenManager;
public class CPPTokenizerContinuationTest {
@ -59,27 +59,27 @@ public class CPPTokenizerContinuationTest {
public void parseWithContinuationCppTokenManager() throws Exception {
String code = load("cpp_with_continuation.cpp");
CppTokenManager tokenManager = new CppTokenManager(new StringReader(code));
List<Token> tokens = new ArrayList<>();
List<JavaccToken> tokens = new ArrayList<>();
Token token = (Token) tokenManager.getNextToken();
while (!token.image.isEmpty()) {
JavaccToken token = (JavaccToken) tokenManager.getNextToken();
while (!token.getImage().isEmpty()) {
tokens.add(token);
token = (Token) tokenManager.getNextToken();
token = (JavaccToken) tokenManager.getNextToken();
}
assertEquals(51, tokens.size());
assertToken(tokens.get(2), "ab", 8, 12, 9, 1);
assertToken(tokens.get(22), "\"2 Hello, world!\\n\"", 18, 16, 19, 9);
assertToken(tokens.get(2), "ab", 8, 12, 9, 2);
assertToken(tokens.get(22), "\"2 Hello, world!\\n\"", 18, 16, 19, 10);
}
private void assertToken(Token token, String image, int beginLine, int beginColumn, int endLine, int endColumn) {
assertEquals(image, token.image);
assertEquals(beginLine, token.beginLine);
assertEquals(beginColumn, token.beginColumn);
assertEquals(endLine, token.endLine);
assertEquals(endColumn, token.endColumn);
private void assertToken(JavaccToken token, String image, int beginLine, int beginColumn, int endLine, int endColumn) {
assertEquals(image, token.getImage());
assertEquals(beginLine, token.getBeginLine());
assertEquals(beginColumn, token.getBeginColumn());
assertEquals(endLine, token.getEndLine());
assertEquals(endColumn, token.getEndColumn());
}
@Test

4
pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/CppCharStreamTest.java → pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java
View File

@ -1,8 +1,8 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp;
package net.sourceforge.pmd.lang.cpp.ast;
import static org.junit.Assert.assertEquals;