diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml
index ab7238f975..52c7b7c313 100644
--- a/pmd-apex-jorje/pom.xml
+++ b/pmd-apex-jorje/pom.xml
@@ -81,7 +81,7 @@
com.google.guava
guava
- 26.0-jre
+ 26.0-android
com.google.j2objc
diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml
index 80aab40792..80d980a398 100644
--- a/pmd-core/pom.xml
+++ b/pmd-core/pom.xml
@@ -186,6 +186,12 @@
system-rules
test
+
+ com.google.guava
+ guava
+ 26.0-android
+ compile
+
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
index 747619b5f3..67cb75937c 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
@@ -4,10 +4,16 @@
package net.sourceforge.pmd.cpd.token.internal;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.LinkedList;
+
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
+import com.google.common.collect.AbstractIterator;
+
/**
* A generic filter for PMD token managers that allows to use comments
* to enable / disable analysis of parts of the stream
@@ -15,7 +21,10 @@ import net.sourceforge.pmd.lang.ast.GenericToken;
public abstract class BaseTokenFilter implements TokenFilter {
private final TokenManager tokenManager;
+ private final LinkedList unprocessedTokens; // NOPMD - used both as Queue and List
+ private final Iterable remainingTokens;
private boolean discardingSuppressing;
+ private T currentToken;
/**
* Creates a new BaseTokenFilter
@@ -23,13 +32,21 @@ public abstract class BaseTokenFilter implements TokenFi
*/
public BaseTokenFilter(final TokenManager tokenManager) {
this.tokenManager = tokenManager;
+ this.unprocessedTokens = new LinkedList<>();
+ this.remainingTokens = new RemainingTokens();
}
@Override
public final T getNextToken() {
- T currentToken = (T) tokenManager.getNextToken();
+ currentToken = null;
+ if (!unprocessedTokens.isEmpty()) {
+ currentToken = unprocessedTokens.poll();
+ return currentToken;
+ }
+ currentToken = (T) tokenManager.getNextToken();
while (!shouldStopProcessing(currentToken)) {
analyzeToken(currentToken);
+ analyzeTokens(currentToken, remainingTokens);
processCPDSuppression(currentToken);
if (!isDiscarding()) {
@@ -73,6 +90,18 @@ public abstract class BaseTokenFilter implements TokenFi
// noop
}
+ /**
+ * Extension point for subclasses to analyze all tokens (before filtering)
+ * and update internal status to decide on custom discard rules.
+ *
+ * @param currentToken The token to be analyzed
+ * @param remainingTokens All upcoming tokens
+ * @see #isLanguageSpecificDiscarding()
+ */
+ protected void analyzeTokens(final T currentToken, final Iterable remainingTokens) {
+ // noop
+ }
+
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
@@ -90,4 +119,42 @@ public abstract class BaseTokenFilter implements TokenFi
*/
protected abstract boolean shouldStopProcessing(T currentToken);
+ private class RemainingTokens implements Iterable {
+
+ @Override
+ public Iterator iterator() {
+ return new RemainingTokensIterator(currentToken);
+ }
+
+ private class RemainingTokensIterator extends AbstractIterator implements Iterator {
+
+ int index = 0; // index of next element
+ T startToken;
+
+ RemainingTokensIterator(final T startToken) {
+ this.startToken = startToken;
+ }
+
+ @Override
+ protected T computeNext() {
+ assert index >= 0;
+ if (startToken != currentToken) { // NOPMD - intentional check for reference equality
+ throw new ConcurrentModificationException("Using iterator after next token has been requested.");
+ }
+ if (index < unprocessedTokens.size()) {
+ return unprocessedTokens.get(index++);
+ } else {
+ final T nextToken = (T) tokenManager.getNextToken();
+ if (shouldStopProcessing(nextToken)) {
+ return endOfData();
+ }
+ index++;
+ unprocessedTokens.add(nextToken);
+ return nextToken;
+ }
+ }
+
+ }
+ }
+
}