phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

C# tokenizer is now Antlr-based.

Browse Source 
This is based on the Antlr grammar from https://github.com/antlr/grammars-v4/tree/master/csharp.

This adds column information for C# and fixes #2139.
This commit is contained in:
Maikel Steneker 2020-02-12 11:19:20 +01:00
parent 659e709f79
commit bdfbfae231
5 changed files with 1433 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java Normal file
View File

@ -0,0 +1,190 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd.token.internal;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.junit.Test;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.GenericToken;
import com.google.common.collect.ImmutableList;
public class BaseTokenFilterTest {
class StringToken implements GenericToken {
private final String text;
StringToken(final String text) {
this.text = text;
}
@Override
public GenericToken getNext() {
return null;
}
@Override
public GenericToken getPreviousComment() {
return null;
}
@Override
public String getImage() {
return text;
}
@Override
public int getBeginLine() {
return 0;
}
@Override
public int getEndLine() {
return 0;
}
@Override
public int getBeginColumn() {
return 0;
}
@Override
public int getEndColumn() {
return 0;
}
}
class StringTokenManager implements TokenManager {
Iterator<String> iterator = ImmutableList.of("a", "b", "c").iterator();
@Override
public Object getNextToken() {
if (iterator.hasNext()) {
return new StringToken(iterator.next());
} else {
return null;
}
}
@Override
public void setFileName(final String fileName) {
}
}
class DummyTokenFilter<T extends GenericToken> extends BaseTokenFilter<T> {
Iterable<T> remainingTokens;
DummyTokenFilter(final TokenManager tokenManager) {
super(tokenManager);
}
@Override
protected boolean shouldStopProcessing(final T currentToken) {
return currentToken == null;
}
@Override
protected void analyzeTokens(final T currentToken, final Iterable<T> remainingTokens) {
this.remainingTokens = remainingTokens;
}
public Iterable getRemainingTokens() {
return remainingTokens;
}
}
@Test
public void testRemainingTokensFunctionality1() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValFirstIt = (StringToken) it1.next();
final StringToken firstValSecondIt = (StringToken) it2.next();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken secondValFirstIt = (StringToken) it1.next();
assertFalse(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken secondValSecondIt = (StringToken) it2.next();
assertFalse(it2.hasNext());
assertEquals("b", firstValFirstIt.getImage());
assertEquals("b", firstValSecondIt.getImage());
assertEquals("c", secondValFirstIt.getImage());
assertEquals("c", secondValSecondIt.getImage());
}
@Test
public void testRemainingTokensFunctionality2() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
assertTrue(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValFirstIt = (StringToken) it1.next();
assertTrue(it1.hasNext());
final StringToken secondValFirstIt = (StringToken) it1.next();
assertFalse(it1.hasNext());
assertTrue(it2.hasNext());
final StringToken firstValSecondIt = (StringToken) it2.next();
assertTrue(it2.hasNext());
final StringToken secondValSecondIt = (StringToken) it2.next();
assertFalse(it2.hasNext());
assertEquals("b", firstValFirstIt.getImage());
assertEquals("b", firstValSecondIt.getImage());
assertEquals("c", secondValFirstIt.getImage());
assertEquals("c", secondValSecondIt.getImage());
}
@Test(expected = NoSuchElementException.class)
public void testRemainingTokensFunctionality3() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final Iterator it2 = iterable.iterator();
it1.next();
it1.next();
it2.next();
it2.next();
it1.next();
}
@Test(expected = ConcurrentModificationException.class)
public void testRemainingTokensFunctionality4() {
final TokenManager tokenManager = new StringTokenManager();
final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
final GenericToken firstToken = tokenFilter.getNextToken();
assertEquals("a", firstToken.getImage());
final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
final Iterator it1 = iterable.iterator();
final GenericToken secondToken = tokenFilter.getNextToken();
assertEquals("b", secondToken.getImage());
it1.next();
}
}

10
pmd-cs/pom.xml
View File

@ -12,6 +12,11 @@
<build>
<plugins>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
@ -23,6 +28,7 @@
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>net.sourceforge.pmd</groupId>
@ -32,10 +38,6 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>

1105
pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4 Normal file
View File

File diff suppressed because it is too large Load Diff

387
pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java
View File

File diff suppressed because it is too large Load Diff

23
pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java
View File

@ -12,6 +12,8 @@ import java.util.List;
import org.junit.Before;
import org.junit.Test;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class CsTokenizerTest {
private CsTokenizer tokenizer;
@ -46,7 +48,7 @@ public class CsTokenizerTest {
public void testSimpleClassMethodMultipleLines() {
tokenizer.tokenize(toSourceCode("class Foo {\n" + " public String foo(int a) {\n" + " int i = a;\n"
+ " return \"x\" + a;\n" + " }\n" + "}"), tokens);
assertEquals(22, tokens.size());
assertEquals(24, tokens.size());
List<TokenEntry> tokenList = tokens.getTokens();
assertEquals(1, tokenList.get(0).getBeginLine());
assertEquals(2, tokenList.get(4).getBeginLine());
@ -56,13 +58,12 @@ public class CsTokenizerTest {
@Test
public void testStrings() {
tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
assertEquals(5, tokens.size());
assertEquals(6, tokens.size());
}
@Test
@Test(expected = TokenMgrError.class)
public void testOpenString() {
tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
assertEquals(5, tokens.size());
}
@Test
@ -91,7 +92,7 @@ public class CsTokenizerTest {
+ " a++; \n" + " a /= 3e2; \n" + " float f = -3.1; \n" + " f *= 2; \n"
+ " bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n" + " }\n" + "}"),
tokens);
assertEquals(50, tokens.size());
assertEquals(57, tokens.size());
}
@Test
@ -119,8 +120,9 @@ public class CsTokenizerTest {
public void testIgnoreUsingDirectives() {
tokenizer.setIgnoreUsings(true);
tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens);
assertEquals(1, tokens.size());
assertNotEquals("using", tokens.getTokens().get(0).toString());
assertEquals(2, tokens.size());
assertEquals(TokenEntry.EOF, tokens.getTokens().get(0));
}
@Test
@ -132,6 +134,15 @@ public class CsTokenizerTest {
assertEquals("using", tokens.getTokens().get(0).toString());
}
@Test
public void testUsingVarStatementsAreNotIgnored() {
tokenizer.setIgnoreUsings(true);
tokenizer.tokenize(toSourceCode(
"using var font1 = new Font(\"Arial\", 10.0f);\n" + " byte charset = font1.GdiCharSet;\n"),
tokens);
assertEquals("using", tokens.getTokens().get(0).toString());
}
private SourceCode toSourceCode(String source) {
return new SourceCode(new SourceCode.StringCodeLoader(source));
}