Deprecate AbstractTokenizer

pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java

@@ -7,11 +7,17 @@ package net.sourceforge.pmd.cpd;
 import java.util.List;
 import java.util.Locale;
 
+import net.sourceforge.pmd.annotation.DeprecatedUntil700;
+
 /**
  *
  * @author Zev Blut zb@ubit.com
  * @author Romain PELISSE belaran@gmail.com
+ *
+ * @deprecated Use an {@link AnyTokenizer} instead, it's basically as powerful.
  */
+@Deprecated
+@DeprecatedUntil700
 public abstract class AbstractTokenizer implements Tokenizer {
 
     // FIXME depending on subclasses to assign local vars is rather fragile -

pmd-perl/src/main/java/net/sourceforge/pmd/cpd/PerlTokenizer.java

@@ -6,6 +6,10 @@ package net.sourceforge.pmd.cpd;
 
 import java.util.ArrayList;
 
+/**
+ * @deprecated Replaced by an {@link AnyTokenizer}. Use {@link PerlLanguage#getTokenizer()} anyway
+ */
+@Deprecated
 public class PerlTokenizer extends AbstractTokenizer {
     public PerlTokenizer() {
         this.stringToken = new ArrayList<>();