do it in c++

pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java

@@ -24,12 +24,12 @@ import net.sourceforge.pmd.util.IOUtil;
 
 public abstract class JavaCCTokenizer implements Tokenizer {
 
-    protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) {
+    protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) throws IOException {
         Reader reader = IOUtil.skipBOM(new CharSequenceReader(sourceCode.getCodeBuffer()));
         return makeLexerImpl(makeCharStream(reader));
     }
 
-    protected CharStream makeCharStream(Reader sourceCode) {
+    protected CharStream makeCharStream(Reader sourceCode) throws IOException {
         return CharStreamFactory.simpleCharStream(sourceCode);
     }
 

pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java

@@ -6,13 +6,17 @@ package net.sourceforge.pmd.cpd;
 
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.Reader;
 import java.io.StringReader;
 import java.util.Properties;
 
 import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.cpp.ast.CppTokenManager;
+import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
+import net.sourceforge.pmd.lang.cpp.ast.CppCharStream;
+import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds;
 import net.sourceforge.pmd.util.IOUtil;
 
 /**
@@ -71,13 +75,21 @@ public class CPPTokenizer extends JavaCCTokenizer {
         }
     }
 
+
     @Override
-    protected TokenManager getLexerForSource(SourceCode sourceCode) {
-        try {
-            StringBuilder buffer = sourceCode.getCodeBuffer();
-            return new CppTokenManager(IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString()))));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+    protected CharStream makeCharStream(Reader sourceCode) {
+        return CppCharStream.newCppCharStream(sourceCode);
+    }
+
+    @Override
+    protected TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode) {
+        return CppTokenKinds.newTokenManager(sourceCode);
+    }
+
+    @Override
+    protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) throws IOException {
+        Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(sourceCode.getCodeBuffer().toString())));
+        CharStream charStream = makeCharStream(reader);
+        return makeLexerImpl(charStream);
     }
 }

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java

@@ -20,7 +20,7 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream;
  *
  * @author Andreas Dangel
  */
-class CppCharStream extends SimpleCharStream {
+public class CppCharStream extends SimpleCharStream {
 
     private static final Pattern CONTINUATION = Pattern.compile("\\\\\\n|\\\\\\r\\n");
     private static final char BACKSLASH = '\\';

pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppTokenManager.java

@@ -1,33 +0,0 @@
-/*
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.cpp.ast;
-
-import java.io.Reader;
-
-import net.sourceforge.pmd.annotation.InternalApi;
-import net.sourceforge.pmd.lang.TokenManager;
-
-/**
- * C++ Token Manager implementation.
- */
-@InternalApi
-public final class CppTokenManager implements TokenManager {
-    private final CppParserImplTokenManager tokenManager;
-
-    /**
-     * Creates a new C++ Token Manager from the given source code.
-     *
-     * @param source the source code
-     */
-    public CppTokenManager(Reader source) {
-        tokenManager = new CppParserImplTokenManager(CppCharStream.newCppCharStream(source));
-    }
-
-    @Override
-    public Object getNextToken() {
-        return tokenManager.getNextToken();
-    }
-
-}

pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerContinuationTest.java

@@ -8,7 +8,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
-import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -19,8 +18,8 @@ import org.junit.Test;
 
 import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.cpd.SourceCode.StringCodeLoader;
+import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
-import net.sourceforge.pmd.lang.cpp.ast.CppTokenManager;
 
 public class CPPTokenizerContinuationTest {
 
@@ -58,13 +57,13 @@ public class CPPTokenizerContinuationTest {
     @Test
     public void parseWithContinuationCppTokenManager() throws Exception {
         String code = load("cpp_with_continuation.cpp");
-        CppTokenManager tokenManager = new CppTokenManager(new StringReader(code));
+        TokenManager<JavaccToken> tokenManager = new CPPTokenizer().getLexerForSource(new SourceCode(new StringCodeLoader(code)));
         List<JavaccToken> tokens = new ArrayList<>();
 
-        JavaccToken token = (JavaccToken) tokenManager.getNextToken();
+        JavaccToken token = tokenManager.getNextToken();
         while (!token.getImage().isEmpty()) {
             tokens.add(token);
-            token = (JavaccToken) tokenManager.getNextToken();
+            token = tokenManager.getNextToken();
         }
 
         assertEquals(51, tokens.size());