C# tokenizer is now Antlr-based.

This is based on the Antlr grammar from https://github.com/antlr/grammars-v4/tree/master/csharp.

This adds column information for C# and fixes #2139.

pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java

@@ -0,0 +1,190 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd.token.internal;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.junit.Test;
+
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
+
+import com.google.common.collect.ImmutableList;
+
+public class BaseTokenFilterTest {
+
+    class StringToken implements GenericToken {
+
+        private final String text;
+
+        StringToken(final String text) {
+            this.text = text;
+        }
+
+        @Override
+        public GenericToken getNext() {
+            return null;
+        }
+
+        @Override
+        public GenericToken getPreviousComment() {
+            return null;
+        }
+
+        @Override
+        public String getImage() {
+            return text;
+        }
+
+        @Override
+        public int getBeginLine() {
+            return 0;
+        }
+
+        @Override
+        public int getEndLine() {
+            return 0;
+        }
+
+        @Override
+        public int getBeginColumn() {
+            return 0;
+        }
+
+        @Override
+        public int getEndColumn() {
+            return 0;
+        }
+    }
+
+    class StringTokenManager implements TokenManager {
+
+        Iterator<String> iterator = ImmutableList.of("a", "b", "c").iterator();
+
+        @Override
+        public Object getNextToken() {
+            if (iterator.hasNext()) {
+                return new StringToken(iterator.next());
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public void setFileName(final String fileName) {
+        }
+    }
+
+    class DummyTokenFilter<T extends GenericToken> extends BaseTokenFilter<T> {
+
+        Iterable<T> remainingTokens;
+
+        DummyTokenFilter(final TokenManager tokenManager) {
+            super(tokenManager);
+        }
+
+        @Override
+        protected boolean shouldStopProcessing(final T currentToken) {
+            return currentToken == null;
+        }
+
+        @Override
+        protected void analyzeTokens(final T currentToken, final Iterable<T> remainingTokens) {
+            this.remainingTokens = remainingTokens;
+        }
+
+        public Iterable getRemainingTokens() {
+            return remainingTokens;
+        }
+    }
+
+    @Test
+    public void testRemainingTokensFunctionality1() {
+        final TokenManager tokenManager = new StringTokenManager();
+        final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
+        final GenericToken firstToken = tokenFilter.getNextToken();
+        assertEquals("a", firstToken.getImage());
+        final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
+        final Iterator it1 = iterable.iterator();
+        final Iterator it2 = iterable.iterator();
+        assertTrue(it1.hasNext());
+        assertTrue(it2.hasNext());
+        final StringToken firstValFirstIt = (StringToken) it1.next();
+        final StringToken firstValSecondIt = (StringToken) it2.next();
+        assertTrue(it1.hasNext());
+        assertTrue(it2.hasNext());
+        final StringToken secondValFirstIt = (StringToken) it1.next();
+        assertFalse(it1.hasNext());
+        assertTrue(it2.hasNext());
+        final StringToken secondValSecondIt = (StringToken) it2.next();
+        assertFalse(it2.hasNext());
+        assertEquals("b", firstValFirstIt.getImage());
+        assertEquals("b", firstValSecondIt.getImage());
+        assertEquals("c", secondValFirstIt.getImage());
+        assertEquals("c", secondValSecondIt.getImage());
+    }
+
+    @Test
+    public void testRemainingTokensFunctionality2() {
+        final TokenManager tokenManager = new StringTokenManager();
+        final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
+        final GenericToken firstToken = tokenFilter.getNextToken();
+        assertEquals("a", firstToken.getImage());
+        final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
+        final Iterator it1 = iterable.iterator();
+        final Iterator it2 = iterable.iterator();
+        assertTrue(it1.hasNext());
+        assertTrue(it2.hasNext());
+        final StringToken firstValFirstIt = (StringToken) it1.next();
+        assertTrue(it1.hasNext());
+        final StringToken secondValFirstIt = (StringToken) it1.next();
+        assertFalse(it1.hasNext());
+        assertTrue(it2.hasNext());
+        final StringToken firstValSecondIt = (StringToken) it2.next();
+        assertTrue(it2.hasNext());
+        final StringToken secondValSecondIt = (StringToken) it2.next();
+        assertFalse(it2.hasNext());
+        assertEquals("b", firstValFirstIt.getImage());
+        assertEquals("b", firstValSecondIt.getImage());
+        assertEquals("c", secondValFirstIt.getImage());
+        assertEquals("c", secondValSecondIt.getImage());
+    }
+
+    @Test(expected = NoSuchElementException.class)
+    public void testRemainingTokensFunctionality3() {
+        final TokenManager tokenManager = new StringTokenManager();
+        final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
+        final GenericToken firstToken = tokenFilter.getNextToken();
+        assertEquals("a", firstToken.getImage());
+        final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
+        final Iterator it1 = iterable.iterator();
+        final Iterator it2 = iterable.iterator();
+        it1.next();
+        it1.next();
+        it2.next();
+        it2.next();
+        it1.next();
+    }
+
+    @Test(expected = ConcurrentModificationException.class)
+    public void testRemainingTokensFunctionality4() {
+        final TokenManager tokenManager = new StringTokenManager();
+        final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager);
+        final GenericToken firstToken = tokenFilter.getNextToken();
+        assertEquals("a", firstToken.getImage());
+        final Iterable<StringToken> iterable = tokenFilter.getRemainingTokens();
+        final Iterator it1 = iterable.iterator();
+        final GenericToken secondToken = tokenFilter.getNextToken();
+        assertEquals("b", secondToken.getImage());
+        it1.next();
+    }
+
+}

pmd-cs/pom.xml

@@ -12,6 +12,11 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-maven-plugin</artifactId>
+            </plugin>
+
             <plugin>
                 <artifactId>maven-resources-plugin</artifactId>
                 <configuration>
@@ -23,6 +28,7 @@
             </plugin>
         </plugins>
     </build>
+
     <dependencies>
         <dependency>
             <groupId>net.sourceforge.pmd</groupId>
@@ -32,10 +38,6 @@
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-        </dependency>
 
         <dependency>
             <groupId>junit</groupId>

pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java

@@ -12,6 +12,8 @@ import java.util.List;
 import org.junit.Before;
 import org.junit.Test;
 
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
+
 public class CsTokenizerTest {
 
     private CsTokenizer tokenizer;
@@ -46,7 +48,7 @@ public class CsTokenizerTest {
     public void testSimpleClassMethodMultipleLines() {
         tokenizer.tokenize(toSourceCode("class Foo {\n" + "  public String foo(int a) {\n" + "    int i = a;\n"
                 + "    return \"x\" + a;\n" + "  }\n" + "}"), tokens);
-        assertEquals(22, tokens.size());
+        assertEquals(24, tokens.size());
         List<TokenEntry> tokenList = tokens.getTokens();
         assertEquals(1, tokenList.get(0).getBeginLine());
         assertEquals(2, tokenList.get(4).getBeginLine());
@@ -56,13 +58,12 @@ public class CsTokenizerTest {
     @Test
     public void testStrings() {
         tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
-        assertEquals(5, tokens.size());
+        assertEquals(6, tokens.size());
     }
 
-    @Test
+    @Test(expected = TokenMgrError.class)
     public void testOpenString() {
         tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
-        assertEquals(5, tokens.size());
     }
 
     @Test
@@ -91,7 +92,7 @@ public class CsTokenizerTest {
                                 + "    a++; \n" + "    a /= 3e2; \n" + "    float f = -3.1; \n" + "    f *= 2; \n"
                                 + "    bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n" + "  }\n" + "}"),
                         tokens);
-        assertEquals(50, tokens.size());
+        assertEquals(57, tokens.size());
     }
 
     @Test
@@ -119,8 +120,9 @@ public class CsTokenizerTest {
     public void testIgnoreUsingDirectives() {
         tokenizer.setIgnoreUsings(true);
         tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens);
+        assertEquals(1, tokens.size());
         assertNotEquals("using", tokens.getTokens().get(0).toString());
-        assertEquals(2, tokens.size());
+        assertEquals(TokenEntry.EOF, tokens.getTokens().get(0));
     }
 
     @Test
@@ -132,6 +134,15 @@ public class CsTokenizerTest {
         assertEquals("using", tokens.getTokens().get(0).toString());
     }
 
+    @Test
+    public void testUsingVarStatementsAreNotIgnored() {
+        tokenizer.setIgnoreUsings(true);
+        tokenizer.tokenize(toSourceCode(
+                "using var font1 = new Font(\"Arial\", 10.0f);\n" + "  byte charset = font1.GdiCharSet;\n"),
+                tokens);
+        assertEquals("using", tokens.getTokens().get(0).toString());
+    }
+
     private SourceCode toSourceCode(String source) {
         return new SourceCode(new SourceCode.StringCodeLoader(source));
     }