hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

[cpp] Add custom CppCharStream for line continuation support

Browse Source 
Fixes #448
This commit is contained in:
Andreas Dangel
2017-07-07 22:44:11 +02:00
parent 37f4be1a35
commit dbc2d02895
4 changed files with 113 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppCharStream.java Normal file
View File

@@ -0,0 +1,62 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp;
import java.io.IOException;
import java.io.Reader;
import java.util.regex.Pattern;
import net.sourceforge.pmd.lang.ast.SimpleCharStream;
/**
* A SimpleCharStream, that supports the continuation of lines via backslash+newline,
* which is used in C/C++.
*
* @author Andreas Dangel
*/
public class CppCharStream extends SimpleCharStream {
private static final Pattern CONTINUATION = Pattern.compile("\\\\\\n|\\\\\\r\\n");
private static final char BACKSLASH = '\\';
private static final char NEWLINE = '\n';
private static final char CARRIAGE_RETURN = '\r';
public CppCharStream(Reader dstream) {
super(dstream);
}
@Override
public char readChar() throws IOException {
char c = super.readChar();
if (c == BACKSLASH) {
char c1 = super.readChar();
if (c1 == NEWLINE) {
c = super.readChar();
} else if (c1 == CARRIAGE_RETURN) {
char c2 = super.readChar();
if (c2 == NEWLINE) {
c = super.readChar();
} else {
backup(2);
}
} else {
backup(1);
}
}
return c;
}
@Override
public char[] GetSuffix(int len) {
String image = GetImage();
return image.substring(image.length() - len, image.length()).toCharArray();
}
@Override
public String GetImage() {
String image = super.GetImage();
return CONTINUATION.matcher(image).replaceAll("");
}
}

3
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppTokenManager.java
View File

@@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.cpp;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.SimpleCharStream;
import net.sourceforge.pmd.lang.cpp.ast.CppParserTokenManager;
/**
@@ -23,7 +22,7 @@ public class CppTokenManager implements TokenManager {
* the source code
*/
public CppTokenManager(Reader source) {
tokenManager = new CppParserTokenManager(new SimpleCharStream(source));
tokenManager = new CppParserTokenManager(new CppCharStream(source));
}
public Object getNextToken() {

16
pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java
View File

@@ -13,7 +13,6 @@ import java.util.List;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.junit.Ignore;
import org.junit.Test;
import net.sourceforge.pmd.PMD;
@@ -25,35 +24,32 @@ public class CPPTokenizerContinuationTest {
public void parseWithContinuation() throws Exception {
String code = load("cpp_with_continuation.cpp");
Tokens tokens = parse(code);
if (tokens.size() < 53) {
if (tokens.size() < 52) {
printTokens(tokens);
fail("Not enough tokens - probably parsing error");
fail("Not enough tokens - probably parsing error. Tokens: " + tokens.size());
}
assertEquals("static", findByLine(8, tokens).get(0).toString());
assertEquals("int", findByLine(8, tokens).get(1).toString());
// Note: the token should be "ab", but since we skip "\\\n" between a and b,
// we end up with two tokens. -> the tokens are not joined.
// This could lead to false negatives in duplication detection.
// However, this is only a problem, if the continuation character is added *within*
// a token and not at token boundaries.
// special case, if the continuation is *within* a token
// see also test #testContinuationIntraToken
//assertEquals("ab", findByLine(8, tokens).get(2).toString());
assertEquals("ab", findByLine(8, tokens).get(2).toString());
assertEquals("int", findByLine(12, tokens).get(0).toString());
assertEquals("main", findByLine(12, tokens).get(1).toString());
assertEquals("(", findByLine(12, tokens).get(2).toString());
assertEquals(")", findByLine(12, tokens).get(3).toString());
assertEquals("{", findByLine(13, tokens).get(0).toString());
assertEquals("\"world!\\n\"", findByLine(16, tokens).get(0).toString());
assertEquals("}", findByLine(29, tokens).get(0).toString());
}
@Test
@Ignore
public void testContinuationIntraToken() throws Exception {
Tokens tokens = parse(load("cpp_continuation_intra_token.cpp"));
assertEquals(7, tokens.size());
printTokens(tokens);
}
@Test

44
pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/CppCharStreamTest.java Normal file
View File

@@ -0,0 +1,44 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.cpp;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.StringReader;
import org.junit.Test;
public class CppCharStreamTest {
@Test
public void testContinuationUnix() throws IOException {
CppCharStream stream = new CppCharStream(new StringReader("a\\\nb"));
assertStream(stream, "ab");
}
@Test
public void testContinuationWindows() throws IOException {
CppCharStream stream = new CppCharStream(new StringReader("a\\\r\nb"));
assertStream(stream, "ab");
}
@Test
public void testBackup() throws IOException {
CppCharStream stream = new CppCharStream(new StringReader("a\\b\\\rc"));
assertStream(stream, "a\\b\\\rc");
}
private void assertStream(CppCharStream stream, String token) throws IOException {
char c = stream.BeginToken();
assertEquals(token.charAt(0), c);
for (int i = 1; i < token.length(); i++) {
c = stream.readChar();
assertEquals(token.charAt(i), c);
}
assertEquals(token, stream.GetImage());
assertEquals(token, new String(stream.GetSuffix(token.length())));
}
}