phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix cpd

Browse Source
This commit is contained in:
Clément Fournier
2022-04-02 15:33:28 +02:00
parent b2b81784ff
commit 474deca0e2
12 changed files with 50 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -6,11 +6,9 @@ package net.sourceforge.pmd.cpd;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Properties;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
@ -19,7 +17,7 @@ import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.cpp.ast.CppCharStream;
import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;
import net.sourceforge.pmd.util.IOUtil;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
* The C++ tokenizer.
@ -78,7 +76,7 @@ public class CPPTokenizer extends JavaCCTokenizer {
filtered.append(line);
}
// always add a new line to keep the line-numbering
filtered.append(PMD.EOL);
filtered.append(System.lineSeparator());
}
return filtered.toString();
}
@ -86,7 +84,7 @@ public class CPPTokenizer extends JavaCCTokenizer {
@Override
protected CharStream makeCharStream(Reader sourceCode) throws IOException {
protected CharStream makeCharStream(TextDocument sourceCode) {
return CppCharStream.newCppCharStream(sourceCode);
}
@ -97,9 +95,8 @@ public class CPPTokenizer extends JavaCCTokenizer {
@SuppressWarnings("PMD.CloseResource")
@Override
protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) throws IOException {
Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(sourceCode.getCodeBuffer().toString())));
CharStream charStream = makeCharStream(reader);
protected TokenManager<JavaccToken> getLexerForSource(TextDocument sourceCode) {
CharStream charStream = makeCharStream(sourceCode);
return makeLexerImpl(charStream);
}

8
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java
View File

@ -5,15 +5,12 @@
package net.sourceforge.pmd.lang.cpp.ast;
import java.io.IOException;
import java.io.Reader;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
import net.sourceforge.pmd.lang.document.CpdCompat;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
@ -67,9 +64,8 @@ public class CppCharStream extends SimpleCharStream {
return CONTINUATION.matcher(image).replaceAll("");
}
public static CppCharStream newCppCharStream(Reader dstream) throws IOException {
String source = IOUtils.toString(dstream);
JavaccTokenDocument document = new JavaccTokenDocument(TextDocument.readOnlyString(source, CpdCompat.dummyVersion())) {
public static CppCharStream newCppCharStream(TextDocument file) {
JavaccTokenDocument document = new JavaccTokenDocument(file) {
@Override
protected @Nullable String describeKindImpl(int kind) {
return CppTokenKinds.describe(kind);

17
pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java
View File

@ -7,29 +7,38 @@ package net.sourceforge.pmd.lang.cpp.ast;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.StringReader;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.Test;
import net.sourceforge.pmd.lang.document.CpdCompat;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.document.TextFile;
public class CppCharStreamTest {
private @NonNull CppCharStream newCharStream(String code) {
TextDocument tf = TextDocument.readOnlyString(code, TextFile.UNKNOWN_FILENAME, CpdCompat.dummyVersion());
return CppCharStream.newCppCharStream(tf);
}
@Test
public void testContinuationUnix() throws IOException {
CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\nb"));
CppCharStream stream = newCharStream("a\\\nb");
assertStream(stream, "ab");
}
@Test
public void testContinuationWindows() throws IOException {
// note that the \r is normalized to a \n by the TextFile
CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\r\nb"));
CppCharStream stream = newCharStream("a\\\r\nb");
assertStream(stream, "ab");
}
@Test
public void testBackup() throws IOException {
// note that the \r is normalized to a \n by the TextFile
CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\b\\qc"));
CppCharStream stream = newCharStream("a\\b\\qc");
assertStream(stream, "a\\b\\qc");
}