From 659e709f7988a5cda6338d8820bde289a22d5f62 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Wed, 12 Feb 2020 11:12:14 +0100 Subject: [PATCH] Adjusted BaseTokenFilter to allow filtering on more than one token. When filtering tokens, the analyzeToken method can be overriden to access the current token. This can then be used to implement isLanguageSpecificDiscarding. However, it may be desirable to "look ahead" and base the decision of whether to filter or not on multiple tokens. In order to support this new use case, a new extension point analyzeTokens is provided, which not only has access to the current token, but can also iterate over the upcoming tokens. The functionality of iterating over remaining tokens uses Guava for its implementation. Since pmd-core targets Java 7, the Android flavour of Guava is used. In order to stay consistent with pmd-apex-jorje, this has also been adjusted to the Android flavour. For PMD 7.0, the jre flavour can be used instead. --- pmd-apex-jorje/pom.xml | 2 +- pmd-core/pom.xml | 6 ++ .../cpd/token/internal/BaseTokenFilter.java | 69 ++++++++++++++++++- 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index ab7238f975..52c7b7c313 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -81,7 +81,7 @@ com.google.guava guava - 26.0-jre + 26.0-android com.google.j2objc diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 80aab40792..80d980a398 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -186,6 +186,12 @@ system-rules test + + com.google.guava + guava + 26.0-android + compile + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 747619b5f3..67cb75937c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -4,10 +4,16 @@ package net.sourceforge.pmd.cpd.token.internal; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.LinkedList; + import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; +import com.google.common.collect.AbstractIterator; + /** * A generic filter for PMD token managers that allows to use comments * to enable / disable analysis of parts of the stream @@ -15,7 +21,10 @@ import net.sourceforge.pmd.lang.ast.GenericToken; public abstract class BaseTokenFilter implements TokenFilter { private final TokenManager tokenManager; + private final LinkedList unprocessedTokens; // NOPMD - used both as Queue and List + private final Iterable remainingTokens; private boolean discardingSuppressing; + private T currentToken; /** * Creates a new BaseTokenFilter @@ -23,13 +32,21 @@ public abstract class BaseTokenFilter implements TokenFi */ public BaseTokenFilter(final TokenManager tokenManager) { this.tokenManager = tokenManager; + this.unprocessedTokens = new LinkedList<>(); + this.remainingTokens = new RemainingTokens(); } @Override public final T getNextToken() { - T currentToken = (T) tokenManager.getNextToken(); + currentToken = null; + if (!unprocessedTokens.isEmpty()) { + currentToken = unprocessedTokens.poll(); + return currentToken; + } + currentToken = (T) tokenManager.getNextToken(); while (!shouldStopProcessing(currentToken)) { analyzeToken(currentToken); + analyzeTokens(currentToken, remainingTokens); processCPDSuppression(currentToken); if (!isDiscarding()) { @@ -73,6 +90,18 @@ public abstract class BaseTokenFilter implements TokenFi // noop } + /** + * Extension point for subclasses to analyze all tokens (before filtering) + * and update internal status to decide on custom discard rules. + * + * @param currentToken The token to be analyzed + * @param remainingTokens All upcoming tokens + * @see #isLanguageSpecificDiscarding() + */ + protected void analyzeTokens(final T currentToken, final Iterable remainingTokens) { + // noop + } + /** * Extension point for subclasses to indicate tokens are to be filtered. * @@ -90,4 +119,42 @@ public abstract class BaseTokenFilter implements TokenFi */ protected abstract boolean shouldStopProcessing(T currentToken); + private class RemainingTokens implements Iterable { + + @Override + public Iterator iterator() { + return new RemainingTokensIterator(currentToken); + } + + private class RemainingTokensIterator extends AbstractIterator implements Iterator { + + int index = 0; // index of next element + T startToken; + + RemainingTokensIterator(final T startToken) { + this.startToken = startToken; + } + + @Override + protected T computeNext() { + assert index >= 0; + if (startToken != currentToken) { // NOPMD - intentional check for reference equality + throw new ConcurrentModificationException("Using iterator after next token has been requested."); + } + if (index < unprocessedTokens.size()) { + return unprocessedTokens.get(index++); + } else { + final T nextToken = (T) tokenManager.getNextToken(); + if (shouldStopProcessing(nextToken)) { + return endOfData(); + } + index++; + unprocessedTokens.add(nextToken); + return nextToken; + } + } + + } + } + }