From f64acb6c3859a4e59aa0240b3de81dc31cd43f15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= <clement.fournier76@gmail.com>
Date: Fri, 20 Mar 2020 16:41:54 +0100
Subject: [PATCH] python

---
 .../sourceforge/pmd/cpd/PythonTokenizer.java  | 32 ++++++++++---
 .../lang/python/ast/PythonTokenManager.java   | 48 -------------------
 2 files changed, 26 insertions(+), 54 deletions(-)
 delete mode 100644 pmd-python/src/main/java/net/sourceforge/pmd/lang/python/ast/PythonTokenManager.java

diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
index e65c0d80b6..32e166e0cf 100644
--- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
+++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
@@ -4,15 +4,18 @@
 
 package net.sourceforge.pmd.cpd;
 
-import java.io.StringReader;
+import java.io.Reader;
 import java.util.regex.Pattern;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
 import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.CharStream;
+import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
 import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
+import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
 import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds;
-import net.sourceforge.pmd.lang.python.ast.PythonTokenManager;
-import net.sourceforge.pmd.util.IOUtil;
 
 /**
  * The Python tokenizer.
@@ -22,9 +25,26 @@ public class PythonTokenizer extends JavaCCTokenizer {
     private static final Pattern STRING_NL_ESCAPE = Pattern.compile("\\\\\\r?\\n");
 
     @Override
-    protected TokenManager getLexerForSource(SourceCode sourceCode) {
-        StringBuilder buffer = sourceCode.getCodeBuffer();
-        return new PythonTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
+    protected TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode) {
+        return PythonTokenKinds.newTokenManager(sourceCode);
+    }
+
+    @Override
+    protected CharStream makeCharStream(Reader sourceCode) {
+        return CharStreamFactory.simpleCharStream(sourceCode, PythonTokenDocument::new);
+    }
+
+    private static class PythonTokenDocument extends JavaccTokenDocument {
+
+        PythonTokenDocument(String fullText) {
+            super(fullText);
+        }
+
+        @Override
+        protected @Nullable String describeKindImpl(int kind) {
+            return PythonTokenKinds.describe(kind);
+        }
+
     }
 
     @Override
diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/ast/PythonTokenManager.java b/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/ast/PythonTokenManager.java
deleted file mode 100644
index b8d8428bee..0000000000
--- a/pmd-python/src/main/java/net/sourceforge/pmd/lang/python/ast/PythonTokenManager.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.python.ast;
-
-import java.io.Reader;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
-import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument;
-
-/**
- * Python Token Manager implementation.
- */
-public class PythonTokenManager implements TokenManager {
-    private final PythonParserImplTokenManager tokenManager;
-
-    /**
-     * Creates a new Python Token Manager from the given source code.
-     *
-     * @param source
-     *            the source code
-     */
-    public PythonTokenManager(Reader source) {
-        tokenManager = new PythonParserImplTokenManager(CharStreamFactory.simpleCharStream(source, PythonTokenDocument::new));
-    }
-
-    @Override
-    public Object getNextToken() {
-        return tokenManager.getNextToken();
-    }
-
-    private static class PythonTokenDocument extends JavaccTokenDocument {
-
-        PythonTokenDocument(String fullText) {
-            super(fullText);
-        }
-
-        @Override
-        protected @Nullable String describeKindImpl(int kind) {
-            return PythonTokenKinds.describe(kind);
-        }
-
-    }
-}