Adjusted BaseTokenFilter to allow filtering on more than one token.

When filtering tokens, the analyzeToken method can be overriden to access the current token. This can then be used to implement isLanguageSpecificDiscarding. However, it may be desirable to "look ahead" and base the decision of whether to filter or not on multiple tokens. In order to support this new use case, a new extension point analyzeTokens is provided, which not only has access to the current token, but can also iterate over the upcoming tokens.

The functionality of iterating over remaining tokens uses Guava for its implementation. Since pmd-core targets Java 7, the Android flavour of Guava is used. In order to stay consistent with pmd-apex-jorje, this has also been adjusted to the Android flavour. For PMD 7.0, the jre flavour can be used instead.

pmd-apex-jorje/pom.xml

@@ -81,7 +81,7 @@
     <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>26.0-jre</version>
+        <version>26.0-android</version>
     </dependency>
     <dependency>
         <groupId>com.google.j2objc</groupId>

pmd-core/pom.xml

@@ -186,6 +186,12 @@
             <artifactId>system-rules</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>26.0-android</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java

@@ -4,10 +4,16 @@
 
 package net.sourceforge.pmd.cpd.token.internal;
 
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.LinkedList;
+
 import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.GenericToken;
 
+import com.google.common.collect.AbstractIterator;
+
 /**
  * A generic filter for PMD token managers that allows to use comments
  * to enable / disable analysis of parts of the stream
@@ -15,7 +21,10 @@ import net.sourceforge.pmd.lang.ast.GenericToken;
 public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFilter {
 
     private final TokenManager tokenManager;
+    private final LinkedList<T> unprocessedTokens; // NOPMD - used both as Queue and List
+    private final Iterable<T> remainingTokens;
     private boolean discardingSuppressing;
+    private T currentToken;
 
     /**
      * Creates a new BaseTokenFilter
@@ -23,13 +32,21 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
      */
     public BaseTokenFilter(final TokenManager tokenManager) {
         this.tokenManager = tokenManager;
+        this.unprocessedTokens = new LinkedList<>();
+        this.remainingTokens = new RemainingTokens();
     }
 
     @Override
     public final T getNextToken() {
-        T currentToken = (T) tokenManager.getNextToken();
+        currentToken = null;
+        if (!unprocessedTokens.isEmpty()) {
+            currentToken = unprocessedTokens.poll();
+            return currentToken;
+        }
+        currentToken = (T) tokenManager.getNextToken();
         while (!shouldStopProcessing(currentToken)) {
             analyzeToken(currentToken);
+            analyzeTokens(currentToken, remainingTokens);
             processCPDSuppression(currentToken);
 
             if (!isDiscarding()) {
@@ -73,6 +90,18 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
         // noop
     }
 
+    /**
+     * Extension point for subclasses to analyze all tokens (before filtering)
+     * and update internal status to decide on custom discard rules.
+     *
+     * @param currentToken The token to be analyzed
+     * @param remainingTokens All upcoming tokens
+     * @see #isLanguageSpecificDiscarding()
+     */
+    protected void analyzeTokens(final T currentToken, final Iterable<T> remainingTokens) {
+        // noop
+    }
+
     /**
      * Extension point for subclasses to indicate tokens are to be filtered.
      *
@@ -90,4 +119,42 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
      */
     protected abstract boolean shouldStopProcessing(T currentToken);
 
+    private class RemainingTokens implements Iterable<T> {
+
+        @Override
+        public Iterator<T> iterator() {
+            return new RemainingTokensIterator(currentToken);
+        }
+
+        private class RemainingTokensIterator extends AbstractIterator<T> implements Iterator<T> {
+
+            int index = 0; // index of next element
+            T startToken;
+
+            RemainingTokensIterator(final T startToken) {
+                this.startToken = startToken;
+            }
+
+            @Override
+            protected T computeNext() {
+                assert index >= 0;
+                if (startToken != currentToken) { // NOPMD - intentional check for reference equality
+                    throw new ConcurrentModificationException("Using iterator after next token has been requested.");
+                }
+                if (index < unprocessedTokens.size()) {
+                    return unprocessedTokens.get(index++);
+                } else {
+                    final T nextToken = (T) tokenManager.getNextToken();
+                    if (shouldStopProcessing(nextToken)) {
+                        return endOfData();
+                    }
+                    index++;
+                    unprocessedTokens.add(nextToken);
+                    return nextToken;
+                }
+            }
+
+        }
+    }
+
 }