phoedos/pmd
1
1 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pr-2280'

Browse Source 
[cs] CPD: Replace C# tokenizer by an Antlr-based one
This commit is contained in:
Andreas Dangel committed 2020-02-29 11:36:14 +01:00
2 parents 1298dd13e9 b65a6a52ea
commit 90f286a4a1
8 files changed
+1567 -283
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/pages/release_notes.md
+9
View File
@@ -24,6 +24,12 @@ For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designe
In addition to suppressing violation with the `@SuppressWarnings` annotation, Apex now also supports
the suppressions with a `NOPMD` comment. See [Suppressing warnings](pmd_userdocs_suppressing_warnings.html).
#### Improved CPD support for C#
The C# tokenizer is now based on an antlr grammar instead of a manual written tokenizer. This
should give more accurate results and especially fixes the problems with the using statement syntax
(see [#2139](https://github.com/pmd/pmd/issues/2139)).
#### New Rules
* The Rule {% rule "apex/design/CognitiveComplexity" %} (`apex-design`) finds methods and classes
@@ -38,6 +44,8 @@ the suppressions with a `NOPMD` comment. See [Suppressing warnings](pmd_userdocs
* [#2306](https://github.com/pmd/pmd/issues/2306): \[apex] Switch statements are not parsed/supported
* apex-design
* [#2162](https://github.com/pmd/pmd/issues/2162): \[apex] Cognitive Complexity rule
* cs
* [#2139](https://github.com/pmd/pmd/issues/2139): \[cs] CPD doesn't understand alternate using statement syntax with C# 8.0
* doc
* [#2274](https://github.com/pmd/pmd/issues/2274): \[doc] Java API documentation for PMD
* java
@@ -138,6 +146,7 @@ methods on {% jdoc apex::lang.apex.ast.ApexParserVisitor %} and its implementati
* [#2276](https://github.com/pmd/pmd/pull/2276): \[java] AppendCharacterWithCharRule ignore literals in expressions - [Kris Scheibe](https://github.com/kris-scheibe)
* [#2278](https://github.com/pmd/pmd/pull/2278): \[java] fix UnusedImports rule for ambiguous static on-demand imports - [Kris Scheibe](https://github.com/kris-scheibe)
* [#2279](https://github.com/pmd/pmd/pull/2279): \[apex] Add support for suppressing violations using the // NOPMD comment - [Gwilym Kuiper](https://github.com/gwilymatgearset)
* [#2280](https://github.com/pmd/pmd/pull/2280): \[cs] CPD: Replace C# tokenizer by an Antlr-based one - [Maikel Steneker](https://github.com/maikelsteneker)
* [#2297](https://github.com/pmd/pmd/pull/2297): \[apex] Cognitive complexity metrics - [Gwilym Kuiper](https://github.com/gwilymatgearset)
{% endtocmaker %}
pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java
+69 -1
View File
@@ -4,6 +4,12 @@
package net.sourceforge.pmd.cpd.token.internal;
import static net.sourceforge.pmd.internal.util.IteratorUtil.AbstractIterator;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.LinkedList;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
@@ -15,7 +21,10 @@ import net.sourceforge.pmd.lang.ast.GenericToken;
public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFilter {
private final TokenManager tokenManager;
private final LinkedList<T> unprocessedTokens; // NOPMD - used both as Queue and List
private final Iterable<T> remainingTokens;
private boolean discardingSuppressing;
private T currentToken;
/**
* Creates a new BaseTokenFilter
@@ -23,13 +32,21 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
*/
public BaseTokenFilter(final TokenManager tokenManager) {
this.tokenManager = tokenManager;
this.unprocessedTokens = new LinkedList<>();
this.remainingTokens = new RemainingTokens();
}
@Override
public final T getNextToken() {
T currentToken = (T) tokenManager.getNextToken();
currentToken = null;
if (!unprocessedTokens.isEmpty()) {
currentToken = unprocessedTokens.poll();
return currentToken;
}
currentToken = (T) tokenManager.getNextToken();
while (!shouldStopProcessing(currentToken)) {
analyzeToken(currentToken);
analyzeTokens(currentToken, remainingTokens);
processCPDSuppression(currentToken);
if (!isDiscarding()) {
@@ -73,6 +90,18 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
// noop
}
/**
* Extension point for subclasses to analyze all tokens (before filtering)
* and update internal status to decide on custom discard rules.
*
* @param currentToken The token to be analyzed
* @param remainingTokens All upcoming tokens
* @see #isLanguageSpecificDiscarding()
*/
protected void analyzeTokens(final T currentToken, final Iterable<T> remainingTokens) {
// noop
}
/**
* Extension point for subclasses to indicate tokens are to be filtered.
*
@@ -90,4 +119,43 @@ public abstract class BaseTokenFilter<T extends GenericToken> implements TokenFi
*/
protected abstract boolean shouldStopProcessing(T currentToken);
private class RemainingTokens implements Iterable<T> {
@Override
public Iterator<T> iterator() {
return new RemainingTokensIterator(currentToken);
}
private class RemainingTokensIterator extends AbstractIterator<T> implements Iterator<T> {
int index = 0; // index of next element
T startToken;
RemainingTokensIterator(final T startToken) {
this.startToken = startToken;
}
@Override
protected void computeNext() {
assert index >= 0;
if (startToken != currentToken) { // NOPMD - intentional check for reference equality
throw new ConcurrentModificationException("Using iterator after next token has been requested.");
}
if (index < unprocessedTokens.size()) {
setNext(unprocessedTokens.get(index++));
} else {
final T nextToken = (T) tokenManager.getNextToken();
if (shouldStopProcessing(nextToken)) {
done();
return;
}
index++;
unprocessedTokens.add(nextToken);
setNext(nextToken);
}
}
}
}
}
pmd-core/src/main/java/net/sourceforge/pmd/internal/util/IteratorUtil.java
+56
View File
@@ -9,6 +9,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.NoSuchElementException;
/**
@@ -88,4 +89,59 @@ public final class IteratorUtil {
}
};
}
public abstract static class AbstractIterator<T> implements Iterator<T> {
private State state = State.NOT_READY;
private T next = null;
@Override
public boolean hasNext() {
switch (state) {
case DONE:
return false;
case READY:
return true;
default:
state = null;
computeNext();
if (state == null) {
throw new IllegalStateException("Should have called done or setNext");
}
return state == State.READY;
}
}
@Override
public T next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
state = State.NOT_READY;
return next;
}
protected final void setNext(T t) {
next = t;
state = State.READY;
}
protected final void done() {
state = State.DONE;
}
protected abstract void computeNext();
enum State {
READY, NOT_READY, DONE
}
@Deprecated
@Override
public final void remove() {
throw new UnsupportedOperationException();
}
}
}
pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java
+190
View File
@@ -0,0 +1,190 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token.internal;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.junit.Test;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
public class BaseTokenFilterTest {
class StringToken implements GenericToken {
private final String text;
StringToken(final String text) {
this.text = text;
}
@Override
public GenericToken getNext() {
return null;
}
@Override
public GenericToken getPreviousComment() {
return null;
}
@Override
public String getImage() {
return text;
}
@Override
public int getBeginLine() {
return 0;
}
@Override
public int getEndLine() {
return 0;
}
@Override
public int getBeginColumn() {
return 0;
}
@Override
public int getEndColumn() {
return 0;
}
}
class StringTokenManager implements TokenManager {
Iterator<String> iterator = Collections.unmodifiableList(Arrays.asList("a", "b", "c")).iterator();
@Override
public Object getNextToken() {
if (iterator.hasNext()) {
return new StringToken(iterator.next());
} else {
return null;
}
}
@Override
public void setFileName(final String fileName) {
}
}
class DummyTokenFilter<T extends GenericToken> extends BaseTokenFilter<T> {
Iterable<T> remainingTokens;
DummyTokenFilter(final TokenManager tokenManager) {
super(tokenManager);
}
@Override
protected boolean shouldStopProcessing(final T currentToken) {
return currentToken == null;
}
@Override
protected void analyzeTokens(final T currentToken, final Iterable<T> remainingTokens) {
this.remainingTokens = remainingTokens;
}
public Iterable getRemainingTokens() {
return remainingTokens;
}
}
@Test
public void testRemainingTokensFunctionality1() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValFirstIt = (StringToken) it1.next();
final StringToken firstValSecondIt = (StringToken) it2.next();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken secondValFirstIt = (StringToken) it1.next();
assertFalse(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken secondValSecondIt = (StringToken) it2.next();
assertFalse(it2.hasNext());
assertEquals("b", firstValFirstIt.getImage());
assertEquals("b", firstValSecondIt.getImage());
assertEquals("c", secondValFirstIt.getImage());
assertEquals("c", secondValSecondIt.getImage());
}
@Test
public void testRemainingTokensFunctionality2() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValFirstIt = (StringToken) it1.next();
assertTrue(it1.hasNext());
final StringToken secondValFirstIt = (StringToken) it1.next();
assertFalse(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValSecondIt = (StringToken) it2.next();
assertTrue(it2.hasNext());
final StringToken secondValSecondIt = (StringToken) it2.next();
assertFalse(it2.hasNext());
assertEquals("b", firstValFirstIt.getImage());
assertEquals("b", firstValSecondIt.getImage());
assertEquals("c", secondValFirstIt.getImage());
assertEquals("c", secondValSecondIt.getImage());
}
@Test(expected = NoSuchElementException.class)
public void testRemainingTokensFunctionality3() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
it1.next();
it1.next();
it2.next();
it2.next();
it1.next();
}
@Test(expected = ConcurrentModificationException.class)
public void testRemainingTokensFunctionality4() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final GenericToken secondToken = tokenFilter.getNextToken();
assertEquals("b", secondToken.getImage());
it1.next();
}
}
pmd-cs/pom.xml
+6 -4
View File
@@ -12,6 +12,11 @@
<build>
<plugins>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
@@ -23,6 +28,7 @@
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>net.sourceforge.pmd</groupId>
@@ -32,10 +38,6 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4
+1105
View File
File diff suppressed because it is too large. Load diff
pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java
+115 -272
View File
File diff suppressed because it is too large. Load diff
pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java
+17 -6
View File
@@ -12,6 +12,8 @@ import java.util.List;
import org.junit.Before;
import org.junit.Test;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class CsTokenizerTest {
private CsTokenizer tokenizer;
@@ -46,7 +48,7 @@ public class CsTokenizerTest {
public void testSimpleClassMethodMultipleLines() {
tokenizer.tokenize(toSourceCode("class Foo {\n" + " public String foo(int a) {\n" + " int i = a;\n"
+ " return \"x\" + a;\n" + " }\n" + "}"), tokens);
assertEquals(22, tokens.size());
assertEquals(24, tokens.size());
List<TokenEntry> tokenList = tokens.getTokens();
assertEquals(1, tokenList.get(0).getBeginLine());
assertEquals(2, tokenList.get(4).getBeginLine());
@@ -56,13 +58,12 @@ public class CsTokenizerTest {
@Test
public void testStrings() {
tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
assertEquals(5, tokens.size());
assertEquals(6, tokens.size());
}
@Test
@Test(expected = TokenMgrError.class)
public void testOpenString() {
tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
assertEquals(5, tokens.size());
}
@Test
@@ -91,7 +92,7 @@ public class CsTokenizerTest {
+ " a++; \n" + " a /= 3e2; \n" + " float f = -3.1; \n" + " f *= 2; \n"
+ " bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n" + " }\n" + "}"),
tokens);
assertEquals(50, tokens.size());
assertEquals(57, tokens.size());
}
@Test
@@ -119,8 +120,9 @@ public class CsTokenizerTest {
public void testIgnoreUsingDirectives() {
tokenizer.setIgnoreUsings(true);
tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens);
assertEquals(1, tokens.size());
assertNotEquals("using", tokens.getTokens().get(0).toString());
assertEquals(2, tokens.size());
assertEquals(TokenEntry.EOF, tokens.getTokens().get(0));
}
@Test
@@ -132,6 +134,15 @@ public class CsTokenizerTest {
assertEquals("using", tokens.getTokens().get(0).toString());
}
@Test
public void testUsingVarStatementsAreNotIgnored() {
tokenizer.setIgnoreUsings(true);
tokenizer.tokenize(toSourceCode(
"using var font1 = new Font(\"Arial\", 10.0f);\n" + " byte charset = font1.GdiCharSet;\n"),
tokens);
assertEquals("using", tokens.getTokens().get(0).toString());
}
private SourceCode toSourceCode(String source) {
return new SourceCode(new SourceCode.StringCodeLoader(source));
}