Fix cpd

pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDCommandLineInterface.java

@@ -120,6 +120,7 @@ public final class CPDCommandLineInterface {
                 setStatusCodeOrExit(NO_ERRORS_STATUS);
             }
         } catch (IOException | RuntimeException e) {
+            e.printStackTrace();
             LOG.debug(e.toString(), e);
             LOG.error(CliMessages.errorDetectedMessage(1, "CPD"));
             setStatusCodeOrExit(ERROR_STATUS);

pmd-core/src/main/java/net/sourceforge/pmd/cpd/MatchAlgorithm.java

@@ -82,6 +82,7 @@ public class MatchAlgorithm {
                 mark.setLineCount(lineCount);
                 mark.setEndToken(endToken);
                 SourceCode sourceCode = source.get(token.getTokenSrcID());
+                assert sourceCode != null : token.getTokenSrcID() + " is not registered in " + source.keySet();
                 mark.setSourceCode(sourceCode);
             }
         }

pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java

@@ -5,9 +5,6 @@
 package net.sourceforge.pmd.cpd.internal;
 
 import java.io.IOException;
-import java.io.Reader;
-
-import org.apache.commons.io.input.CharSequenceReader;
 
 import net.sourceforge.pmd.cpd.SourceCode;
 import net.sourceforge.pmd.cpd.TokenEntry;
@@ -20,17 +17,17 @@ import net.sourceforge.pmd.lang.ast.CharStream;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
-import net.sourceforge.pmd.util.IOUtil;
+import net.sourceforge.pmd.lang.document.CpdCompat;
+import net.sourceforge.pmd.lang.document.TextDocument;
 
 public abstract class JavaCCTokenizer implements Tokenizer {
 
     @SuppressWarnings("PMD.CloseResource")
-    protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) throws IOException {
-        Reader reader = IOUtil.skipBOM(new CharSequenceReader(sourceCode.getCodeBuffer()));
-        return makeLexerImpl(makeCharStream(reader));
+    protected TokenManager<JavaccToken> getLexerForSource(TextDocument sourceCode) throws IOException {
+        return makeLexerImpl(makeCharStream(sourceCode));
     }
 
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CharStreamFactory.simpleCharStream(sourceCode);
     }
 
@@ -50,8 +47,8 @@ public abstract class JavaCCTokenizer implements Tokenizer {
 
     @Override
     public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException {
-        TokenManager<JavaccToken> tokenManager = getLexerForSource(sourceCode);
-        try {
+        try (TextDocument textDoc = TextDocument.create(CpdCompat.cpdCompat(sourceCode))) {
+            TokenManager<JavaccToken> tokenManager = getLexerForSource(textDoc);
             final TokenFilter<JavaccToken> tokenFilter = getTokenFilter(tokenManager);
             JavaccToken currentToken = tokenFilter.getNextToken();
             while (currentToken != null) {

pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java

@@ -4,14 +4,9 @@
 
 package net.sourceforge.pmd.lang.ast.impl.javacc;
 
-import java.io.IOException;
-import java.io.Reader;
 import java.util.function.Function;
 
-import org.apache.commons.io.IOUtils;
-
 import net.sourceforge.pmd.lang.ast.CharStream;
-import net.sourceforge.pmd.lang.document.CpdCompat;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
 public final class CharStreamFactory {
@@ -23,36 +18,32 @@ public final class CharStreamFactory {
     /**
      * A char stream that doesn't perform any escape translation.
      */
-    public static CharStream simpleCharStream(Reader input) throws IOException {
+    public static CharStream simpleCharStream(TextDocument input) {
         return simpleCharStream(input, JavaccTokenDocument::new);
     }
 
     /**
      * A char stream that doesn't perform any escape translation.
      */
-    public static CharStream simpleCharStream(Reader input,
-                                              Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker)
-        throws IOException {
-        String source = IOUtils.toString(input);
-        JavaccTokenDocument document = documentMaker.apply(TextDocument.readOnlyString(source, CpdCompat.dummyVersion()));
+    public static CharStream simpleCharStream(TextDocument input,
+                                              Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker) {
+        JavaccTokenDocument document = documentMaker.apply(input);
         return new SimpleCharStream(document);
     }
 
     /**
      * A char stream that translates java unicode sequences.
      */
-    public static CharStream javaCharStream(Reader input) throws IOException {
+    public static CharStream javaCharStream(TextDocument input) {
         return javaCharStream(input, JavaccTokenDocument::new);
     }
 
     /**
      * A char stream that translates java unicode sequences.
      */
-    public static CharStream javaCharStream(Reader input, Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker)
-        throws IOException {
-        String source = IOUtils.toString(input);
-        JavaccTokenDocument tokens = documentMaker.apply(TextDocument.readOnlyString(source, CpdCompat.dummyVersion()));
-        return new JavaCharStream(tokens);
+    public static CharStream javaCharStream(TextDocument input, Function<? super TextDocument, ? extends JavaccTokenDocument> documentMaker) {
+        JavaccTokenDocument document = documentMaker.apply(input);
+        return new JavaCharStream(document);
     }
 
 }

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -6,11 +6,9 @@ package net.sourceforge.pmd.cpd;
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.Reader;
 import java.io.StringReader;
 import java.util.Properties;
 
-import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.cpd.token.TokenFilter;
@@ -19,7 +17,7 @@ import net.sourceforge.pmd.lang.ast.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
 import net.sourceforge.pmd.lang.cpp.ast.CppCharStream;
 import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;
-import net.sourceforge.pmd.util.IOUtil;
+import net.sourceforge.pmd.lang.document.TextDocument;
 
 /**
  * The C++ tokenizer.
@@ -78,7 +76,7 @@ public class CPPTokenizer extends JavaCCTokenizer {
                     filtered.append(line);
                 }
                 // always add a new line to keep the line-numbering
-                filtered.append(PMD.EOL);
+                filtered.append(System.lineSeparator());
             }
             return filtered.toString();
         }
@@ -86,7 +84,7 @@ public class CPPTokenizer extends JavaCCTokenizer {
 
 
     @Override
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CppCharStream.newCppCharStream(sourceCode);
     }
 
@@ -97,9 +95,8 @@ public class CPPTokenizer extends JavaCCTokenizer {
 
     @SuppressWarnings("PMD.CloseResource")
     @Override
-    protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) throws IOException {
-        Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(sourceCode.getCodeBuffer().toString())));
-        CharStream charStream = makeCharStream(reader);
+    protected TokenManager<JavaccToken> getLexerForSource(TextDocument sourceCode) {
+        CharStream charStream = makeCharStream(sourceCode);
         return makeLexerImpl(charStream);
     }
 

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java

@@ -5,15 +5,12 @@
 package net.sourceforge.pmd.lang.cpp.ast;
 
 import java.io.IOException;
-import java.io.Reader;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.IOUtils;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
 import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
-import net.sourceforge.pmd.lang.document.CpdCompat;
 import net.sourceforge.pmd.lang.document.TextDocument;
 
 /**
@@ -67,9 +64,8 @@ public class CppCharStream extends SimpleCharStream {
         return CONTINUATION.matcher(image).replaceAll("");
     }
 
-    public static CppCharStream newCppCharStream(Reader dstream) throws IOException {
-        String source = IOUtils.toString(dstream);
-        JavaccTokenDocument document = new JavaccTokenDocument(TextDocument.readOnlyString(source, CpdCompat.dummyVersion())) {
+    public static CppCharStream newCppCharStream(TextDocument file) {
+        JavaccTokenDocument document = new JavaccTokenDocument(file) {
             @Override
             protected @Nullable String describeKindImpl(int kind) {
                 return CppTokenKinds.describe(kind);

pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java

@@ -7,29 +7,38 @@ package net.sourceforge.pmd.lang.cpp.ast;
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
-import java.io.StringReader;
 
+import org.checkerframework.checker.nullness.qual.NonNull;
 import org.junit.Test;
 
+import net.sourceforge.pmd.lang.document.CpdCompat;
+import net.sourceforge.pmd.lang.document.TextDocument;
+import net.sourceforge.pmd.lang.document.TextFile;
+
 public class CppCharStreamTest {
 
+    private @NonNull CppCharStream newCharStream(String code) {
+        TextDocument tf = TextDocument.readOnlyString(code, TextFile.UNKNOWN_FILENAME, CpdCompat.dummyVersion());
+        return CppCharStream.newCppCharStream(tf);
+    }
+
     @Test
     public void testContinuationUnix() throws IOException {
-        CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\nb"));
+        CppCharStream stream = newCharStream("a\\\nb");
         assertStream(stream, "ab");
     }
 
     @Test
     public void testContinuationWindows() throws IOException {
         // note that the \r is normalized to a \n by the TextFile
-        CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\r\nb"));
+        CppCharStream stream = newCharStream("a\\\r\nb");
         assertStream(stream, "ab");
     }
 
     @Test
     public void testBackup() throws IOException {
         // note that the \r is normalized to a \n by the TextFile
-        CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\b\\qc"));
+        CppCharStream stream = newCharStream("a\\b\\qc");
         assertStream(stream, "a\\b\\qc");
     }
 

pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java

@@ -5,7 +5,6 @@
 package net.sourceforge.pmd.cpd;
 
 import java.io.IOException;
-import java.io.Reader;
 import java.util.Deque;
 import java.util.LinkedList;
 import java.util.Properties;
@@ -17,6 +16,7 @@ import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
+import net.sourceforge.pmd.lang.document.TextDocument;
 import net.sourceforge.pmd.lang.java.ast.InternalApiBridge;
 import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds;
 
@@ -44,7 +44,7 @@ public class JavaTokenizer extends JavaCCTokenizer {
     }
 
     @Override
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CharStreamFactory.javaCharStream(sourceCode, InternalApiBridge::javaTokenDoc);
     }
 

pmd-java/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java

@@ -4,6 +4,10 @@
 
 package net.sourceforge.pmd.cpd;
 
+import static net.sourceforge.pmd.cli.BaseCLITest.containsPattern;
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 import java.util.regex.Pattern;
 
 import org.junit.Assert;
@@ -89,8 +93,8 @@ public class CPDCommandLineInterfaceTest extends BaseCPDCLITest {
         System.setProperty("file.encoding", origEncoding);
 
         String out = getOutput();
-        Assert.assertTrue(out.startsWith("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
-        Assert.assertTrue(Pattern.compile("System\\.out\\.println\\([ij] \\+ \"ä\"\\);").matcher(out).find());
+        assertThat(out, startsWith("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
+        assertThat(out, containsPattern("System\\.out\\.println\\([ij] \\+ \"ä\"\\);"));
         Assert.assertEquals(4, Integer.parseInt(System.getProperty(CPDCommandLineInterface.STATUS_CODE_PROPERTY)));
     }
 

pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java

@@ -4,14 +4,12 @@
 
 package net.sourceforge.pmd.cpd;
 
-import java.io.IOException;
-import java.io.Reader;
-
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
+import net.sourceforge.pmd.lang.document.TextDocument;
 import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds;
 
 public class JSPTokenizer extends JavaCCTokenizer {
@@ -22,7 +20,7 @@ public class JSPTokenizer extends JavaCCTokenizer {
     }
 
     @Override
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CharStreamFactory.javaCharStream(sourceCode);
     }
 }

pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java

@@ -4,8 +4,6 @@
 
 package net.sourceforge.pmd.cpd;
 
-import java.io.IOException;
-import java.io.Reader;
 import java.util.regex.Pattern;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -32,7 +30,7 @@ public class PythonTokenizer extends JavaCCTokenizer {
     }
 
     @Override
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CharStreamFactory.simpleCharStream(sourceCode, PythonTokenDocument::new);
     }
 

pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java

@@ -4,14 +4,12 @@
 
 package net.sourceforge.pmd.cpd;
 
-import java.io.IOException;
-import java.io.Reader;
-
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.CharStream;
 import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
+import net.sourceforge.pmd.lang.document.TextDocument;
 import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds;
 
 /**
@@ -25,7 +23,7 @@ public class VfTokenizer extends JavaCCTokenizer {
     }
 
     @Override
-    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
+    protected CharStream makeCharStream(TextDocument sourceCode) {
         return CharStreamFactory.javaCharStream(sourceCode);
     }
 }