From 659e709f7988a5cda6338d8820bde289a22d5f62 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Wed, 12 Feb 2020 11:12:14 +0100 Subject: [PATCH 1/4] Adjusted BaseTokenFilter to allow filtering on more than one token. When filtering tokens, the analyzeToken method can be overriden to access the current token. This can then be used to implement isLanguageSpecificDiscarding. However, it may be desirable to "look ahead" and base the decision of whether to filter or not on multiple tokens. In order to support this new use case, a new extension point analyzeTokens is provided, which not only has access to the current token, but can also iterate over the upcoming tokens. The functionality of iterating over remaining tokens uses Guava for its implementation. Since pmd-core targets Java 7, the Android flavour of Guava is used. In order to stay consistent with pmd-apex-jorje, this has also been adjusted to the Android flavour. For PMD 7.0, the jre flavour can be used instead. --- pmd-apex-jorje/pom.xml | 2 +- pmd-core/pom.xml | 6 ++ .../cpd/token/internal/BaseTokenFilter.java | 69 ++++++++++++++++++- 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index ab7238f975..52c7b7c313 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -81,7 +81,7 @@ com.google.guava guava - 26.0-jre + 26.0-android com.google.j2objc diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 80aab40792..80d980a398 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -186,6 +186,12 @@ system-rules test + + com.google.guava + guava + 26.0-android + compile + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 747619b5f3..67cb75937c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -4,10 +4,16 @@ package net.sourceforge.pmd.cpd.token.internal; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.LinkedList; + import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; +import com.google.common.collect.AbstractIterator; + /** * A generic filter for PMD token managers that allows to use comments * to enable / disable analysis of parts of the stream @@ -15,7 +21,10 @@ import net.sourceforge.pmd.lang.ast.GenericToken; public abstract class BaseTokenFilter implements TokenFilter { private final TokenManager tokenManager; + private final LinkedList unprocessedTokens; // NOPMD - used both as Queue and List + private final Iterable remainingTokens; private boolean discardingSuppressing; + private T currentToken; /** * Creates a new BaseTokenFilter @@ -23,13 +32,21 @@ public abstract class BaseTokenFilter implements TokenFi */ public BaseTokenFilter(final TokenManager tokenManager) { this.tokenManager = tokenManager; + this.unprocessedTokens = new LinkedList<>(); + this.remainingTokens = new RemainingTokens(); } @Override public final T getNextToken() { - T currentToken = (T) tokenManager.getNextToken(); + currentToken = null; + if (!unprocessedTokens.isEmpty()) { + currentToken = unprocessedTokens.poll(); + return currentToken; + } + currentToken = (T) tokenManager.getNextToken(); while (!shouldStopProcessing(currentToken)) { analyzeToken(currentToken); + analyzeTokens(currentToken, remainingTokens); processCPDSuppression(currentToken); if (!isDiscarding()) { @@ -73,6 +90,18 @@ public abstract class BaseTokenFilter implements TokenFi // noop } + /** + * Extension point for subclasses to analyze all tokens (before filtering) + * and update internal status to decide on custom discard rules. + * + * @param currentToken The token to be analyzed + * @param remainingTokens All upcoming tokens + * @see #isLanguageSpecificDiscarding() + */ + protected void analyzeTokens(final T currentToken, final Iterable remainingTokens) { + // noop + } + /** * Extension point for subclasses to indicate tokens are to be filtered. * @@ -90,4 +119,42 @@ public abstract class BaseTokenFilter implements TokenFi */ protected abstract boolean shouldStopProcessing(T currentToken); + private class RemainingTokens implements Iterable { + + @Override + public Iterator iterator() { + return new RemainingTokensIterator(currentToken); + } + + private class RemainingTokensIterator extends AbstractIterator implements Iterator { + + int index = 0; // index of next element + T startToken; + + RemainingTokensIterator(final T startToken) { + this.startToken = startToken; + } + + @Override + protected T computeNext() { + assert index >= 0; + if (startToken != currentToken) { // NOPMD - intentional check for reference equality + throw new ConcurrentModificationException("Using iterator after next token has been requested."); + } + if (index < unprocessedTokens.size()) { + return unprocessedTokens.get(index++); + } else { + final T nextToken = (T) tokenManager.getNextToken(); + if (shouldStopProcessing(nextToken)) { + return endOfData(); + } + index++; + unprocessedTokens.add(nextToken); + return nextToken; + } + } + + } + } + } From bdfbfae231ce437f6c9a6a67e7a313b6295a9632 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Wed, 12 Feb 2020 11:19:20 +0100 Subject: [PATCH 2/4] C# tokenizer is now Antlr-based. This is based on the Antlr grammar from https://github.com/antlr/grammars-v4/tree/master/csharp. This adds column information for C# and fixes #2139. --- .../token/internal/BaseTokenFilterTest.java | 190 +++ pmd-cs/pom.xml | 10 +- .../pmd/lang/cs/antlr4/CSharpLexer.g4 | 1105 +++++++++++++++++ .../net/sourceforge/pmd/cpd/CsTokenizer.java | 387 ++---- .../sourceforge/pmd/cpd/CsTokenizerTest.java | 23 +- 5 files changed, 1433 insertions(+), 282 deletions(-) create mode 100644 pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java create mode 100644 pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4 diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java new file mode 100644 index 0000000000..21da8ccc19 --- /dev/null +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java @@ -0,0 +1,190 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd.token.internal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.junit.Test; + +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.GenericToken; + +import com.google.common.collect.ImmutableList; + +public class BaseTokenFilterTest { + + class StringToken implements GenericToken { + + private final String text; + + StringToken(final String text) { + this.text = text; + } + + @Override + public GenericToken getNext() { + return null; + } + + @Override + public GenericToken getPreviousComment() { + return null; + } + + @Override + public String getImage() { + return text; + } + + @Override + public int getBeginLine() { + return 0; + } + + @Override + public int getEndLine() { + return 0; + } + + @Override + public int getBeginColumn() { + return 0; + } + + @Override + public int getEndColumn() { + return 0; + } + } + + class StringTokenManager implements TokenManager { + + Iterator iterator = ImmutableList.of("a", "b", "c").iterator(); + + @Override + public Object getNextToken() { + if (iterator.hasNext()) { + return new StringToken(iterator.next()); + } else { + return null; + } + } + + @Override + public void setFileName(final String fileName) { + } + } + + class DummyTokenFilter extends BaseTokenFilter { + + Iterable remainingTokens; + + DummyTokenFilter(final TokenManager tokenManager) { + super(tokenManager); + } + + @Override + protected boolean shouldStopProcessing(final T currentToken) { + return currentToken == null; + } + + @Override + protected void analyzeTokens(final T currentToken, final Iterable remainingTokens) { + this.remainingTokens = remainingTokens; + } + + public Iterable getRemainingTokens() { + return remainingTokens; + } + } + + @Test + public void testRemainingTokensFunctionality1() { + final TokenManager tokenManager = new StringTokenManager(); + final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager); + final GenericToken firstToken = tokenFilter.getNextToken(); + assertEquals("a", firstToken.getImage()); + final Iterable iterable = tokenFilter.getRemainingTokens(); + final Iterator it1 = iterable.iterator(); + final Iterator it2 = iterable.iterator(); + assertTrue(it1.hasNext()); + assertTrue(it2.hasNext()); + final StringToken firstValFirstIt = (StringToken) it1.next(); + final StringToken firstValSecondIt = (StringToken) it2.next(); + assertTrue(it1.hasNext()); + assertTrue(it2.hasNext()); + final StringToken secondValFirstIt = (StringToken) it1.next(); + assertFalse(it1.hasNext()); + assertTrue(it2.hasNext()); + final StringToken secondValSecondIt = (StringToken) it2.next(); + assertFalse(it2.hasNext()); + assertEquals("b", firstValFirstIt.getImage()); + assertEquals("b", firstValSecondIt.getImage()); + assertEquals("c", secondValFirstIt.getImage()); + assertEquals("c", secondValSecondIt.getImage()); + } + + @Test + public void testRemainingTokensFunctionality2() { + final TokenManager tokenManager = new StringTokenManager(); + final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager); + final GenericToken firstToken = tokenFilter.getNextToken(); + assertEquals("a", firstToken.getImage()); + final Iterable iterable = tokenFilter.getRemainingTokens(); + final Iterator it1 = iterable.iterator(); + final Iterator it2 = iterable.iterator(); + assertTrue(it1.hasNext()); + assertTrue(it2.hasNext()); + final StringToken firstValFirstIt = (StringToken) it1.next(); + assertTrue(it1.hasNext()); + final StringToken secondValFirstIt = (StringToken) it1.next(); + assertFalse(it1.hasNext()); + assertTrue(it2.hasNext()); + final StringToken firstValSecondIt = (StringToken) it2.next(); + assertTrue(it2.hasNext()); + final StringToken secondValSecondIt = (StringToken) it2.next(); + assertFalse(it2.hasNext()); + assertEquals("b", firstValFirstIt.getImage()); + assertEquals("b", firstValSecondIt.getImage()); + assertEquals("c", secondValFirstIt.getImage()); + assertEquals("c", secondValSecondIt.getImage()); + } + + @Test(expected = NoSuchElementException.class) + public void testRemainingTokensFunctionality3() { + final TokenManager tokenManager = new StringTokenManager(); + final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager); + final GenericToken firstToken = tokenFilter.getNextToken(); + assertEquals("a", firstToken.getImage()); + final Iterable iterable = tokenFilter.getRemainingTokens(); + final Iterator it1 = iterable.iterator(); + final Iterator it2 = iterable.iterator(); + it1.next(); + it1.next(); + it2.next(); + it2.next(); + it1.next(); + } + + @Test(expected = ConcurrentModificationException.class) + public void testRemainingTokensFunctionality4() { + final TokenManager tokenManager = new StringTokenManager(); + final DummyTokenFilter tokenFilter = new DummyTokenFilter(tokenManager); + final GenericToken firstToken = tokenFilter.getNextToken(); + assertEquals("a", firstToken.getImage()); + final Iterable iterable = tokenFilter.getRemainingTokens(); + final Iterator it1 = iterable.iterator(); + final GenericToken secondToken = tokenFilter.getNextToken(); + assertEquals("b", secondToken.getImage()); + it1.next(); + } + +} diff --git a/pmd-cs/pom.xml b/pmd-cs/pom.xml index 551f9fd4b3..39679d6150 100644 --- a/pmd-cs/pom.xml +++ b/pmd-cs/pom.xml @@ -12,6 +12,11 @@ + + org.antlr + antlr4-maven-plugin + + maven-resources-plugin @@ -23,6 +28,7 @@ + net.sourceforge.pmd @@ -32,10 +38,6 @@ commons-io commons-io - - org.apache.commons - commons-lang3 - junit diff --git a/pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4 b/pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4 new file mode 100644 index 0000000000..37428cbc72 --- /dev/null +++ b/pmd-cs/src/main/antlr4/net/sourceforge/pmd/lang/cs/antlr4/CSharpLexer.g4 @@ -0,0 +1,1105 @@ +// Eclipse Public License - v 1.0, http://www.eclipse.org/legal/epl-v10.html +// Copyright (c) 2013, Christian Wulf (chwchw@gmx.de) +// Copyright (c) 2016-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. + +lexer grammar CSharpLexer; + +@lexer::header +{import java.util.Stack;} + +channels { COMMENTS_CHANNEL, DIRECTIVE } + +@lexer::members +{private int interpolatedStringLevel; +private Stack interpolatedVerbatiums = new Stack(); +private Stack curlyLevels = new Stack(); +private boolean verbatium; +} + +BYTE_ORDER_MARK: '\u00EF\u00BB\u00BF'; + +SINGLE_LINE_DOC_COMMENT: '///' InputCharacter* -> channel(COMMENTS_CHANNEL); +DELIMITED_DOC_COMMENT: '/**' .*? '*/' -> channel(COMMENTS_CHANNEL); +SINGLE_LINE_COMMENT: '//' InputCharacter* -> channel(COMMENTS_CHANNEL); +DELIMITED_COMMENT: '/*' .*? '*/' -> channel(COMMENTS_CHANNEL); + +WHITESPACES: (Whitespace | NewLine)+ -> channel(HIDDEN); +SHARP: '#' -> mode(DIRECTIVE_MODE); + +ABSTRACT: 'abstract'; +ADD: 'add'; +ALIAS: 'alias'; +ARGLIST: '__arglist'; +AS: 'as'; +ASCENDING: 'ascending'; +ASYNC: 'async'; +AWAIT: 'await'; +BASE: 'base'; +BOOL: 'bool'; +BREAK: 'break'; +BY: 'by'; +BYTE: 'byte'; +CASE: 'case'; +CATCH: 'catch'; +CHAR: 'char'; +CHECKED: 'checked'; +CLASS: 'class'; +CONST: 'const'; +CONTINUE: 'continue'; +DECIMAL: 'decimal'; +DEFAULT: 'default'; +DELEGATE: 'delegate'; +DESCENDING: 'descending'; +DO: 'do'; +DOUBLE: 'double'; +DYNAMIC: 'dynamic'; +ELSE: 'else'; +ENUM: 'enum'; +EQUALS: 'equals'; +EVENT: 'event'; +EXPLICIT: 'explicit'; +EXTERN: 'extern'; +FALSE: 'false'; +FINALLY: 'finally'; +FIXED: 'fixed'; +FLOAT: 'float'; +FOR: 'for'; +FOREACH: 'foreach'; +FROM: 'from'; +GET: 'get'; +GOTO: 'goto'; +GROUP: 'group'; +IF: 'if'; +IMPLICIT: 'implicit'; +IN: 'in'; +INT: 'int'; +INTERFACE: 'interface'; +INTERNAL: 'internal'; +INTO: 'into'; +IS: 'is'; +JOIN: 'join'; +LET: 'let'; +LOCK: 'lock'; +LONG: 'long'; +NAMEOF: 'nameof'; +NAMESPACE: 'namespace'; +NEW: 'new'; +NULL: 'null'; +OBJECT: 'object'; +ON: 'on'; +OPERATOR: 'operator'; +ORDERBY: 'orderby'; +OUT: 'out'; +OVERRIDE: 'override'; +PARAMS: 'params'; +PARTIAL: 'partial'; +PRIVATE: 'private'; +PROTECTED: 'protected'; +PUBLIC: 'public'; +READONLY: 'readonly'; +REF: 'ref'; +REMOVE: 'remove'; +RETURN: 'return'; +SBYTE: 'sbyte'; +SEALED: 'sealed'; +SELECT: 'select'; +SET: 'set'; +SHORT: 'short'; +SIZEOF: 'sizeof'; +STACKALLOC: 'stackalloc'; +STATIC: 'static'; +STRING: 'string'; +STRUCT: 'struct'; +SWITCH: 'switch'; +THIS: 'this'; +THROW: 'throw'; +TRUE: 'true'; +TRY: 'try'; +TYPEOF: 'typeof'; +UINT: 'uint'; +ULONG: 'ulong'; +UNCHECKED: 'unchecked'; +UNSAFE: 'unsafe'; +USHORT: 'ushort'; +USING: 'using'; +VAR: 'var'; +VIRTUAL: 'virtual'; +VOID: 'void'; +VOLATILE: 'volatile'; +WHEN: 'when'; +WHERE: 'where'; +WHILE: 'while'; +YIELD: 'yield'; + +//B.1.6 Identifiers +// must be defined after all keywords so the first branch (Available_identifier) does not match keywords +// https://msdn.microsoft.com/en-us/library/aa664670(v=vs.71).aspx +IDENTIFIER: '@'? IdentifierOrKeyword; + +//B.1.8 Literals +// 0.Equals() would be parsed as an invalid real (1. branch) causing a lexer error +LITERAL_ACCESS: [0-9]+ IntegerTypeSuffix? '.' '@'? IdentifierOrKeyword; +INTEGER_LITERAL: [0-9]+ IntegerTypeSuffix?; +HEX_INTEGER_LITERAL: '0' [xX] HexDigit+ IntegerTypeSuffix?; +REAL_LITERAL: [0-9]* '.' [0-9]+ ExponentPart? [FfDdMm]? | [0-9]+ ([FfDdMm] | ExponentPart [FfDdMm]?); + +CHARACTER_LITERAL: '\'' (~['\\\r\n\u0085\u2028\u2029] | CommonCharacter) '\''; +REGULAR_STRING: '"' (~["\\\r\n\u0085\u2028\u2029] | CommonCharacter)* '"'; +VERBATIUM_STRING: '@"' (~'"' | '""')* '"'; +INTERPOLATED_REGULAR_STRING_START: '$"' + { interpolatedStringLevel++; interpolatedVerbatiums.push(false); verbatium = false; } -> pushMode(INTERPOLATION_STRING); +INTERPOLATED_VERBATIUM_STRING_START: '$@"' + { interpolatedStringLevel++; interpolatedVerbatiums.push(true); verbatium = true; } -> pushMode(INTERPOLATION_STRING); + +//B.1.9 Operators And Punctuators +OPEN_BRACE: '{' +{ +if (interpolatedStringLevel > 0) +{ + curlyLevels.push(curlyLevels.pop() + 1); +}}; +CLOSE_BRACE: '}' +{ +if (interpolatedStringLevel > 0) +{ + curlyLevels.push(curlyLevels.pop() - 1); + if (curlyLevels.peek() == 0) + { + curlyLevels.pop(); + skip(); + popMode(); + } +} +}; +OPEN_BRACKET: '['; +CLOSE_BRACKET: ']'; +OPEN_PARENS: '('; +CLOSE_PARENS: ')'; +DOT: '.'; +COMMA: ','; +COLON: ':' +{ +if (interpolatedStringLevel > 0) +{ + int ind = 1; + boolean switchToFormatString = true; + while ((char)_input.LA(ind) != '}') + { + if (_input.LA(ind) == ':' || _input.LA(ind) == ')') + { + switchToFormatString = false; + break; + } + ind++; + } + if (switchToFormatString) + { + mode(INTERPOLATION_FORMAT); + } +} +}; +SEMICOLON: ';'; +PLUS: '+'; +MINUS: '-'; +STAR: '*'; +DIV: '/'; +PERCENT: '%'; +AMP: '&'; +BITWISE_OR: '|'; +CARET: '^'; +BANG: '!'; +TILDE: '~'; +ASSIGNMENT: '='; +LT: '<'; +GT: '>'; +INTERR: '?'; +DOUBLE_COLON: '::'; +OP_COALESCING: '??'; +OP_INC: '++'; +OP_DEC: '--'; +OP_AND: '&&'; +OP_OR: '||'; +OP_PTR: '->'; +OP_EQ: '=='; +OP_NE: '!='; +OP_LE: '<='; +OP_GE: '>='; +OP_ADD_ASSIGNMENT: '+='; +OP_SUB_ASSIGNMENT: '-='; +OP_MULT_ASSIGNMENT: '*='; +OP_DIV_ASSIGNMENT: '/='; +OP_MOD_ASSIGNMENT: '%='; +OP_AND_ASSIGNMENT: '&='; +OP_OR_ASSIGNMENT: '|='; +OP_XOR_ASSIGNMENT: '^='; +OP_LEFT_SHIFT: '<<'; +OP_LEFT_SHIFT_ASSIGNMENT: '<<='; + +// https://msdn.microsoft.com/en-us/library/dn961160.aspx +mode INTERPOLATION_STRING; + +DOUBLE_CURLY_INSIDE: '{{'; +OPEN_BRACE_INSIDE: '{' { curlyLevels.push(1); } -> skip, pushMode(DEFAULT_MODE); +REGULAR_CHAR_INSIDE: { !verbatium }? SimpleEscapeSequence; +VERBATIUM_DOUBLE_QUOTE_INSIDE: { verbatium }? '""'; +DOUBLE_QUOTE_INSIDE: '"' { interpolatedStringLevel--; interpolatedVerbatiums.pop(); + verbatium = (interpolatedVerbatiums.size() > 0 ? interpolatedVerbatiums.peek() : false); } -> popMode; +REGULAR_STRING_INSIDE: { !verbatium }? ~('{' | '\\' | '"')+; +VERBATIUM_INSIDE_STRING: { verbatium }? ~('{' | '"')+; + +mode INTERPOLATION_FORMAT; + +DOUBLE_CURLY_CLOSE_INSIDE: '}}' -> type(FORMAT_STRING); +CLOSE_BRACE_INSIDE: '}' { curlyLevels.pop(); } -> skip, popMode; +FORMAT_STRING: ~'}'+; + +mode DIRECTIVE_MODE; + +DIRECTIVE_WHITESPACES: Whitespace+ -> channel(HIDDEN); +DIGITS: [0-9]+ -> channel(DIRECTIVE); +DIRECTIVE_TRUE: 'true' -> channel(DIRECTIVE), type(TRUE); +DIRECTIVE_FALSE: 'false' -> channel(DIRECTIVE), type(FALSE); +DEFINE: 'define' -> channel(DIRECTIVE); +UNDEF: 'undef' -> channel(DIRECTIVE); +DIRECTIVE_IF: 'if' -> channel(DIRECTIVE), type(IF); +ELIF: 'elif' -> channel(DIRECTIVE); +DIRECTIVE_ELSE: 'else' -> channel(DIRECTIVE), type(ELSE); +ENDIF: 'endif' -> channel(DIRECTIVE); +LINE: 'line' -> channel(DIRECTIVE); +ERROR: 'error' Whitespace+ -> channel(DIRECTIVE), mode(DIRECTIVE_TEXT); +WARNING: 'warning' Whitespace+ -> channel(DIRECTIVE), mode(DIRECTIVE_TEXT); +REGION: 'region' Whitespace* -> channel(DIRECTIVE), mode(DIRECTIVE_TEXT); +ENDREGION: 'endregion' Whitespace* -> channel(DIRECTIVE), mode(DIRECTIVE_TEXT); +PRAGMA: 'pragma' Whitespace+ -> channel(DIRECTIVE), mode(DIRECTIVE_TEXT); +DIRECTIVE_DEFAULT: 'default' -> channel(DIRECTIVE), type(DEFAULT); +DIRECTIVE_HIDDEN: 'hidden' -> channel(DIRECTIVE); +DIRECTIVE_OPEN_PARENS: '(' -> channel(DIRECTIVE), type(OPEN_PARENS); +DIRECTIVE_CLOSE_PARENS: ')' -> channel(DIRECTIVE), type(CLOSE_PARENS); +DIRECTIVE_BANG: '!' -> channel(DIRECTIVE), type(BANG); +DIRECTIVE_OP_EQ: '==' -> channel(DIRECTIVE), type(OP_EQ); +DIRECTIVE_OP_NE: '!=' -> channel(DIRECTIVE), type(OP_NE); +DIRECTIVE_OP_AND: '&&' -> channel(DIRECTIVE), type(OP_AND); +DIRECTIVE_OP_OR: '||' -> channel(DIRECTIVE), type(OP_OR); +DIRECTIVE_STRING: '"' ~('"' | [\r\n\u0085\u2028\u2029])* '"' -> channel(DIRECTIVE), type(STRING); +CONDITIONAL_SYMBOL: IdentifierOrKeyword -> channel(DIRECTIVE); +DIRECTIVE_SINGLE_LINE_COMMENT: '//' ~[\r\n\u0085\u2028\u2029]* -> channel(COMMENTS_CHANNEL), type(SINGLE_LINE_COMMENT); +DIRECTIVE_NEW_LINE: NewLine -> channel(DIRECTIVE), mode(DEFAULT_MODE); + +mode DIRECTIVE_TEXT; + +TEXT: ~[\r\n\u0085\u2028\u2029]+ -> channel(DIRECTIVE); +TEXT_NEW_LINE: NewLine -> channel(DIRECTIVE), type(DIRECTIVE_NEW_LINE), mode(DEFAULT_MODE); + +// Fragments + +fragment InputCharacter: ~[\r\n\u0085\u2028\u2029]; + +fragment NewLineCharacter + : '\u000D' //'' + | '\u000A' //'' + | '\u0085' //'' + | '\u2028' //'' + | '\u2029' //'' + ; + +fragment IntegerTypeSuffix: [lL]? [uU] | [uU]? [lL]; +fragment ExponentPart: [eE] ('+' | '-')? [0-9]+; + +fragment CommonCharacter + : SimpleEscapeSequence + | HexEscapeSequence + | UnicodeEscapeSequence + ; + +fragment SimpleEscapeSequence + : '\\\'' + | '\\"' + | '\\\\' + | '\\0' + | '\\a' + | '\\b' + | '\\f' + | '\\n' + | '\\r' + | '\\t' + | '\\v' + ; + +fragment HexEscapeSequence + : '\\x' HexDigit + | '\\x' HexDigit HexDigit + | '\\x' HexDigit HexDigit HexDigit + | '\\x' HexDigit HexDigit HexDigit HexDigit + ; + +fragment NewLine + : NL + ; + +NL + : '\r\n' | '\r' | '\n' + | '\u0085' // ' + | '\u2028' //'' + | '\u2029' //'' + ; + +fragment Whitespace + : UnicodeClassZS //'' + | '\u0009' //'' + | '\u000B' //'' + | '\u000C' //'
' + ; + +fragment UnicodeClassZS + : '\u0020' // SPACE + | '\u00A0' // NO_BREAK SPACE + | '\u1680' // OGHAM SPACE MARK + | '\u180E' // MONGOLIAN VOWEL SEPARATOR + | '\u2000' // EN QUAD + | '\u2001' // EM QUAD + | '\u2002' // EN SPACE + | '\u2003' // EM SPACE + | '\u2004' // THREE_PER_EM SPACE + | '\u2005' // FOUR_PER_EM SPACE + | '\u2006' // SIX_PER_EM SPACE + | '\u2008' // PUNCTUATION SPACE + | '\u2009' // THIN SPACE + | '\u200A' // HAIR SPACE + | '\u202F' // NARROW NO_BREAK SPACE + | '\u3000' // IDEOGRAPHIC SPACE + | '\u205F' // MEDIUM MATHEMATICAL SPACE + ; + +fragment IdentifierOrKeyword + : IdentifierStartCharacter IdentifierPartCharacter* + ; + +fragment IdentifierStartCharacter + : LetterCharacter + | '_' + ; + +fragment IdentifierPartCharacter + : LetterCharacter + | DecimalDigitCharacter + | ConnectingCharacter + | CombiningCharacter + | FormattingCharacter + ; + +//'' +// WARNING: ignores UnicodeEscapeSequence +fragment LetterCharacter + : UnicodeClassLU + | UnicodeClassLL + | UnicodeClassLT + | UnicodeClassLM + | UnicodeClassLO + | UnicodeClassNL + | UnicodeEscapeSequence + ; + +//'' +// WARNING: ignores UnicodeEscapeSequence +fragment DecimalDigitCharacter + : UnicodeClassND + | UnicodeEscapeSequence + ; + +//'' +// WARNING: ignores UnicodeEscapeSequence +fragment ConnectingCharacter + : UnicodeClassPC + | UnicodeEscapeSequence + ; + +//'' +// WARNING: ignores UnicodeEscapeSequence +fragment CombiningCharacter + : UnicodeClassMN + | UnicodeClassMC + | UnicodeEscapeSequence + ; + +//'' +// WARNING: ignores UnicodeEscapeSequence +fragment FormattingCharacter + : UnicodeClassCF + | UnicodeEscapeSequence + ; + +//B.1.5 Unicode Character Escape Sequences +fragment UnicodeEscapeSequence + : '\\u' HexDigit HexDigit HexDigit HexDigit + | '\\U' HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit + ; + +fragment HexDigit : [0-9] | [A-F] | [a-f]; + +// Unicode character classes +fragment UnicodeClassLU + : '\u0041'..'\u005a' + | '\u00c0'..'\u00d6' + | '\u00d8'..'\u00de' + | '\u0100'..'\u0136' + | '\u0139'..'\u0147' + | '\u014a'..'\u0178' + | '\u0179'..'\u017d' + | '\u0181'..'\u0182' + | '\u0184'..'\u0186' + | '\u0187'..'\u0189' + | '\u018a'..'\u018b' + | '\u018e'..'\u0191' + | '\u0193'..'\u0194' + | '\u0196'..'\u0198' + | '\u019c'..'\u019d' + | '\u019f'..'\u01a0' + | '\u01a2'..'\u01a6' + | '\u01a7'..'\u01a9' + | '\u01ac'..'\u01ae' + | '\u01af'..'\u01b1' + | '\u01b2'..'\u01b3' + | '\u01b5'..'\u01b7' + | '\u01b8'..'\u01bc' + | '\u01c4'..'\u01cd' + | '\u01cf'..'\u01db' + | '\u01de'..'\u01ee' + | '\u01f1'..'\u01f4' + | '\u01f6'..'\u01f8' + | '\u01fa'..'\u0232' + | '\u023a'..'\u023b' + | '\u023d'..'\u023e' + | '\u0241'..'\u0243' + | '\u0244'..'\u0246' + | '\u0248'..'\u024e' + | '\u0370'..'\u0372' + | '\u0376'..'\u037f' + | '\u0386'..'\u0388' + | '\u0389'..'\u038a' + | '\u038c'..'\u038e' + | '\u038f'..'\u0391' + | '\u0392'..'\u03a1' + | '\u03a3'..'\u03ab' + | '\u03cf'..'\u03d2' + | '\u03d3'..'\u03d4' + | '\u03d8'..'\u03ee' + | '\u03f4'..'\u03f7' + | '\u03f9'..'\u03fa' + | '\u03fd'..'\u042f' + | '\u0460'..'\u0480' + | '\u048a'..'\u04c0' + | '\u04c1'..'\u04cd' + | '\u04d0'..'\u052e' + | '\u0531'..'\u0556' + | '\u10a0'..'\u10c5' + | '\u10c7'..'\u10cd' + | '\u1e00'..'\u1e94' + | '\u1e9e'..'\u1efe' + | '\u1f08'..'\u1f0f' + | '\u1f18'..'\u1f1d' + | '\u1f28'..'\u1f2f' + | '\u1f38'..'\u1f3f' + | '\u1f48'..'\u1f4d' + | '\u1f59'..'\u1f5f' + | '\u1f68'..'\u1f6f' + | '\u1fb8'..'\u1fbb' + | '\u1fc8'..'\u1fcb' + | '\u1fd8'..'\u1fdb' + | '\u1fe8'..'\u1fec' + | '\u1ff8'..'\u1ffb' + | '\u2102'..'\u2107' + | '\u210b'..'\u210d' + | '\u2110'..'\u2112' + | '\u2115'..'\u2119' + | '\u211a'..'\u211d' + | '\u2124'..'\u212a' + | '\u212b'..'\u212d' + | '\u2130'..'\u2133' + | '\u213e'..'\u213f' + | '\u2145'..'\u2183' + | '\u2c00'..'\u2c2e' + | '\u2c60'..'\u2c62' + | '\u2c63'..'\u2c64' + | '\u2c67'..'\u2c6d' + | '\u2c6e'..'\u2c70' + | '\u2c72'..'\u2c75' + | '\u2c7e'..'\u2c80' + | '\u2c82'..'\u2ce2' + | '\u2ceb'..'\u2ced' + | '\u2cf2'..'\ua640' + | '\ua642'..'\ua66c' + | '\ua680'..'\ua69a' + | '\ua722'..'\ua72e' + | '\ua732'..'\ua76e' + | '\ua779'..'\ua77d' + | '\ua77e'..'\ua786' + | '\ua78b'..'\ua78d' + | '\ua790'..'\ua792' + | '\ua796'..'\ua7aa' + | '\ua7ab'..'\ua7ad' + | '\ua7b0'..'\ua7b1' + | '\uff21'..'\uff3a' + ; + +fragment UnicodeClassLL + : '\u0061'..'\u007A' + | '\u00b5'..'\u00df' + | '\u00e0'..'\u00f6' + | '\u00f8'..'\u00ff' + | '\u0101'..'\u0137' + | '\u0138'..'\u0148' + | '\u0149'..'\u0177' + | '\u017a'..'\u017e' + | '\u017f'..'\u0180' + | '\u0183'..'\u0185' + | '\u0188'..'\u018c' + | '\u018d'..'\u0192' + | '\u0195'..'\u0199' + | '\u019a'..'\u019b' + | '\u019e'..'\u01a1' + | '\u01a3'..'\u01a5' + | '\u01a8'..'\u01aa' + | '\u01ab'..'\u01ad' + | '\u01b0'..'\u01b4' + | '\u01b6'..'\u01b9' + | '\u01ba'..'\u01bd' + | '\u01be'..'\u01bf' + | '\u01c6'..'\u01cc' + | '\u01ce'..'\u01dc' + | '\u01dd'..'\u01ef' + | '\u01f0'..'\u01f3' + | '\u01f5'..'\u01f9' + | '\u01fb'..'\u0233' + | '\u0234'..'\u0239' + | '\u023c'..'\u023f' + | '\u0240'..'\u0242' + | '\u0247'..'\u024f' + | '\u0250'..'\u0293' + | '\u0295'..'\u02af' + | '\u0371'..'\u0373' + | '\u0377'..'\u037b' + | '\u037c'..'\u037d' + | '\u0390'..'\u03ac' + | '\u03ad'..'\u03ce' + | '\u03d0'..'\u03d1' + | '\u03d5'..'\u03d7' + | '\u03d9'..'\u03ef' + | '\u03f0'..'\u03f3' + | '\u03f5'..'\u03fb' + | '\u03fc'..'\u0430' + | '\u0431'..'\u045f' + | '\u0461'..'\u0481' + | '\u048b'..'\u04bf' + | '\u04c2'..'\u04ce' + | '\u04cf'..'\u052f' + | '\u0561'..'\u0587' + | '\u1d00'..'\u1d2b' + | '\u1d6b'..'\u1d77' + | '\u1d79'..'\u1d9a' + | '\u1e01'..'\u1e95' + | '\u1e96'..'\u1e9d' + | '\u1e9f'..'\u1eff' + | '\u1f00'..'\u1f07' + | '\u1f10'..'\u1f15' + | '\u1f20'..'\u1f27' + | '\u1f30'..'\u1f37' + | '\u1f40'..'\u1f45' + | '\u1f50'..'\u1f57' + | '\u1f60'..'\u1f67' + | '\u1f70'..'\u1f7d' + | '\u1f80'..'\u1f87' + | '\u1f90'..'\u1f97' + | '\u1fa0'..'\u1fa7' + | '\u1fb0'..'\u1fb4' + | '\u1fb6'..'\u1fb7' + | '\u1fbe'..'\u1fc2' + | '\u1fc3'..'\u1fc4' + | '\u1fc6'..'\u1fc7' + | '\u1fd0'..'\u1fd3' + | '\u1fd6'..'\u1fd7' + | '\u1fe0'..'\u1fe7' + | '\u1ff2'..'\u1ff4' + | '\u1ff6'..'\u1ff7' + | '\u210a'..'\u210e' + | '\u210f'..'\u2113' + | '\u212f'..'\u2139' + | '\u213c'..'\u213d' + | '\u2146'..'\u2149' + | '\u214e'..'\u2184' + | '\u2c30'..'\u2c5e' + | '\u2c61'..'\u2c65' + | '\u2c66'..'\u2c6c' + | '\u2c71'..'\u2c73' + | '\u2c74'..'\u2c76' + | '\u2c77'..'\u2c7b' + | '\u2c81'..'\u2ce3' + | '\u2ce4'..'\u2cec' + | '\u2cee'..'\u2cf3' + | '\u2d00'..'\u2d25' + | '\u2d27'..'\u2d2d' + | '\ua641'..'\ua66d' + | '\ua681'..'\ua69b' + | '\ua723'..'\ua72f' + | '\ua730'..'\ua731' + | '\ua733'..'\ua771' + | '\ua772'..'\ua778' + | '\ua77a'..'\ua77c' + | '\ua77f'..'\ua787' + | '\ua78c'..'\ua78e' + | '\ua791'..'\ua793' + | '\ua794'..'\ua795' + | '\ua797'..'\ua7a9' + | '\ua7fa'..'\uab30' + | '\uab31'..'\uab5a' + | '\uab64'..'\uab65' + | '\ufb00'..'\ufb06' + | '\ufb13'..'\ufb17' + | '\uff41'..'\uff5a' + ; + +fragment UnicodeClassLT + : '\u01c5'..'\u01cb' + | '\u01f2'..'\u1f88' + | '\u1f89'..'\u1f8f' + | '\u1f98'..'\u1f9f' + | '\u1fa8'..'\u1faf' + | '\u1fbc'..'\u1fcc' + | '\u1ffc'..'\u1ffc' + ; + +fragment UnicodeClassLM + : '\u02b0'..'\u02c1' + | '\u02c6'..'\u02d1' + | '\u02e0'..'\u02e4' + | '\u02ec'..'\u02ee' + | '\u0374'..'\u037a' + | '\u0559'..'\u0640' + | '\u06e5'..'\u06e6' + | '\u07f4'..'\u07f5' + | '\u07fa'..'\u081a' + | '\u0824'..'\u0828' + | '\u0971'..'\u0e46' + | '\u0ec6'..'\u10fc' + | '\u17d7'..'\u1843' + | '\u1aa7'..'\u1c78' + | '\u1c79'..'\u1c7d' + | '\u1d2c'..'\u1d6a' + | '\u1d78'..'\u1d9b' + | '\u1d9c'..'\u1dbf' + | '\u2071'..'\u207f' + | '\u2090'..'\u209c' + | '\u2c7c'..'\u2c7d' + | '\u2d6f'..'\u2e2f' + | '\u3005'..'\u3031' + | '\u3032'..'\u3035' + | '\u303b'..'\u309d' + | '\u309e'..'\u30fc' + | '\u30fd'..'\u30fe' + | '\ua015'..'\ua4f8' + | '\ua4f9'..'\ua4fd' + | '\ua60c'..'\ua67f' + | '\ua69c'..'\ua69d' + | '\ua717'..'\ua71f' + | '\ua770'..'\ua788' + | '\ua7f8'..'\ua7f9' + | '\ua9cf'..'\ua9e6' + | '\uaa70'..'\uaadd' + | '\uaaf3'..'\uaaf4' + | '\uab5c'..'\uab5f' + | '\uff70'..'\uff9e' + | '\uff9f'..'\uff9f' + ; + +fragment UnicodeClassLO + : '\u00aa'..'\u00ba' + | '\u01bb'..'\u01c0' + | '\u01c1'..'\u01c3' + | '\u0294'..'\u05d0' + | '\u05d1'..'\u05ea' + | '\u05f0'..'\u05f2' + | '\u0620'..'\u063f' + | '\u0641'..'\u064a' + | '\u066e'..'\u066f' + | '\u0671'..'\u06d3' + | '\u06d5'..'\u06ee' + | '\u06ef'..'\u06fa' + | '\u06fb'..'\u06fc' + | '\u06ff'..'\u0710' + | '\u0712'..'\u072f' + | '\u074d'..'\u07a5' + | '\u07b1'..'\u07ca' + | '\u07cb'..'\u07ea' + | '\u0800'..'\u0815' + | '\u0840'..'\u0858' + | '\u08a0'..'\u08b2' + | '\u0904'..'\u0939' + | '\u093d'..'\u0950' + | '\u0958'..'\u0961' + | '\u0972'..'\u0980' + | '\u0985'..'\u098c' + | '\u098f'..'\u0990' + | '\u0993'..'\u09a8' + | '\u09aa'..'\u09b0' + | '\u09b2'..'\u09b6' + | '\u09b7'..'\u09b9' + | '\u09bd'..'\u09ce' + | '\u09dc'..'\u09dd' + | '\u09df'..'\u09e1' + | '\u09f0'..'\u09f1' + | '\u0a05'..'\u0a0a' + | '\u0a0f'..'\u0a10' + | '\u0a13'..'\u0a28' + | '\u0a2a'..'\u0a30' + | '\u0a32'..'\u0a33' + | '\u0a35'..'\u0a36' + | '\u0a38'..'\u0a39' + | '\u0a59'..'\u0a5c' + | '\u0a5e'..'\u0a72' + | '\u0a73'..'\u0a74' + | '\u0a85'..'\u0a8d' + | '\u0a8f'..'\u0a91' + | '\u0a93'..'\u0aa8' + | '\u0aaa'..'\u0ab0' + | '\u0ab2'..'\u0ab3' + | '\u0ab5'..'\u0ab9' + | '\u0abd'..'\u0ad0' + | '\u0ae0'..'\u0ae1' + | '\u0b05'..'\u0b0c' + | '\u0b0f'..'\u0b10' + | '\u0b13'..'\u0b28' + | '\u0b2a'..'\u0b30' + | '\u0b32'..'\u0b33' + | '\u0b35'..'\u0b39' + | '\u0b3d'..'\u0b5c' + | '\u0b5d'..'\u0b5f' + | '\u0b60'..'\u0b61' + | '\u0b71'..'\u0b83' + | '\u0b85'..'\u0b8a' + | '\u0b8e'..'\u0b90' + | '\u0b92'..'\u0b95' + | '\u0b99'..'\u0b9a' + | '\u0b9c'..'\u0b9e' + | '\u0b9f'..'\u0ba3' + | '\u0ba4'..'\u0ba8' + | '\u0ba9'..'\u0baa' + | '\u0bae'..'\u0bb9' + | '\u0bd0'..'\u0c05' + | '\u0c06'..'\u0c0c' + | '\u0c0e'..'\u0c10' + | '\u0c12'..'\u0c28' + | '\u0c2a'..'\u0c39' + | '\u0c3d'..'\u0c58' + | '\u0c59'..'\u0c60' + | '\u0c61'..'\u0c85' + | '\u0c86'..'\u0c8c' + | '\u0c8e'..'\u0c90' + | '\u0c92'..'\u0ca8' + | '\u0caa'..'\u0cb3' + | '\u0cb5'..'\u0cb9' + | '\u0cbd'..'\u0cde' + | '\u0ce0'..'\u0ce1' + | '\u0cf1'..'\u0cf2' + | '\u0d05'..'\u0d0c' + | '\u0d0e'..'\u0d10' + | '\u0d12'..'\u0d3a' + | '\u0d3d'..'\u0d4e' + | '\u0d60'..'\u0d61' + | '\u0d7a'..'\u0d7f' + | '\u0d85'..'\u0d96' + | '\u0d9a'..'\u0db1' + | '\u0db3'..'\u0dbb' + | '\u0dbd'..'\u0dc0' + | '\u0dc1'..'\u0dc6' + | '\u0e01'..'\u0e30' + | '\u0e32'..'\u0e33' + | '\u0e40'..'\u0e45' + | '\u0e81'..'\u0e82' + | '\u0e84'..'\u0e87' + | '\u0e88'..'\u0e8a' + | '\u0e8d'..'\u0e94' + | '\u0e95'..'\u0e97' + | '\u0e99'..'\u0e9f' + | '\u0ea1'..'\u0ea3' + | '\u0ea5'..'\u0ea7' + | '\u0eaa'..'\u0eab' + | '\u0ead'..'\u0eb0' + | '\u0eb2'..'\u0eb3' + | '\u0ebd'..'\u0ec0' + | '\u0ec1'..'\u0ec4' + | '\u0edc'..'\u0edf' + | '\u0f00'..'\u0f40' + | '\u0f41'..'\u0f47' + | '\u0f49'..'\u0f6c' + | '\u0f88'..'\u0f8c' + | '\u1000'..'\u102a' + | '\u103f'..'\u1050' + | '\u1051'..'\u1055' + | '\u105a'..'\u105d' + | '\u1061'..'\u1065' + | '\u1066'..'\u106e' + | '\u106f'..'\u1070' + | '\u1075'..'\u1081' + | '\u108e'..'\u10d0' + | '\u10d1'..'\u10fa' + | '\u10fd'..'\u1248' + | '\u124a'..'\u124d' + | '\u1250'..'\u1256' + | '\u1258'..'\u125a' + | '\u125b'..'\u125d' + | '\u1260'..'\u1288' + | '\u128a'..'\u128d' + | '\u1290'..'\u12b0' + | '\u12b2'..'\u12b5' + | '\u12b8'..'\u12be' + | '\u12c0'..'\u12c2' + | '\u12c3'..'\u12c5' + | '\u12c8'..'\u12d6' + | '\u12d8'..'\u1310' + | '\u1312'..'\u1315' + | '\u1318'..'\u135a' + | '\u1380'..'\u138f' + | '\u13a0'..'\u13f4' + | '\u1401'..'\u166c' + | '\u166f'..'\u167f' + | '\u1681'..'\u169a' + | '\u16a0'..'\u16ea' + | '\u16f1'..'\u16f8' + | '\u1700'..'\u170c' + | '\u170e'..'\u1711' + | '\u1720'..'\u1731' + | '\u1740'..'\u1751' + | '\u1760'..'\u176c' + | '\u176e'..'\u1770' + | '\u1780'..'\u17b3' + | '\u17dc'..'\u1820' + | '\u1821'..'\u1842' + | '\u1844'..'\u1877' + | '\u1880'..'\u18a8' + | '\u18aa'..'\u18b0' + | '\u18b1'..'\u18f5' + | '\u1900'..'\u191e' + | '\u1950'..'\u196d' + | '\u1970'..'\u1974' + | '\u1980'..'\u19ab' + | '\u19c1'..'\u19c7' + | '\u1a00'..'\u1a16' + | '\u1a20'..'\u1a54' + | '\u1b05'..'\u1b33' + | '\u1b45'..'\u1b4b' + | '\u1b83'..'\u1ba0' + | '\u1bae'..'\u1baf' + | '\u1bba'..'\u1be5' + | '\u1c00'..'\u1c23' + | '\u1c4d'..'\u1c4f' + | '\u1c5a'..'\u1c77' + | '\u1ce9'..'\u1cec' + | '\u1cee'..'\u1cf1' + | '\u1cf5'..'\u1cf6' + | '\u2135'..'\u2138' + | '\u2d30'..'\u2d67' + | '\u2d80'..'\u2d96' + | '\u2da0'..'\u2da6' + | '\u2da8'..'\u2dae' + | '\u2db0'..'\u2db6' + | '\u2db8'..'\u2dbe' + | '\u2dc0'..'\u2dc6' + | '\u2dc8'..'\u2dce' + | '\u2dd0'..'\u2dd6' + | '\u2dd8'..'\u2dde' + | '\u3006'..'\u303c' + | '\u3041'..'\u3096' + | '\u309f'..'\u30a1' + | '\u30a2'..'\u30fa' + | '\u30ff'..'\u3105' + | '\u3106'..'\u312d' + | '\u3131'..'\u318e' + | '\u31a0'..'\u31ba' + | '\u31f0'..'\u31ff' + | '\u3400'..'\u4db5' + | '\u4e00'..'\u9fcc' + | '\ua000'..'\ua014' + | '\ua016'..'\ua48c' + | '\ua4d0'..'\ua4f7' + | '\ua500'..'\ua60b' + | '\ua610'..'\ua61f' + | '\ua62a'..'\ua62b' + | '\ua66e'..'\ua6a0' + | '\ua6a1'..'\ua6e5' + | '\ua7f7'..'\ua7fb' + | '\ua7fc'..'\ua801' + | '\ua803'..'\ua805' + | '\ua807'..'\ua80a' + | '\ua80c'..'\ua822' + | '\ua840'..'\ua873' + | '\ua882'..'\ua8b3' + | '\ua8f2'..'\ua8f7' + | '\ua8fb'..'\ua90a' + | '\ua90b'..'\ua925' + | '\ua930'..'\ua946' + | '\ua960'..'\ua97c' + | '\ua984'..'\ua9b2' + | '\ua9e0'..'\ua9e4' + | '\ua9e7'..'\ua9ef' + | '\ua9fa'..'\ua9fe' + | '\uaa00'..'\uaa28' + | '\uaa40'..'\uaa42' + | '\uaa44'..'\uaa4b' + | '\uaa60'..'\uaa6f' + | '\uaa71'..'\uaa76' + | '\uaa7a'..'\uaa7e' + | '\uaa7f'..'\uaaaf' + | '\uaab1'..'\uaab5' + | '\uaab6'..'\uaab9' + | '\uaaba'..'\uaabd' + | '\uaac0'..'\uaac2' + | '\uaadb'..'\uaadc' + | '\uaae0'..'\uaaea' + | '\uaaf2'..'\uab01' + | '\uab02'..'\uab06' + | '\uab09'..'\uab0e' + | '\uab11'..'\uab16' + | '\uab20'..'\uab26' + | '\uab28'..'\uab2e' + | '\uabc0'..'\uabe2' + | '\uac00'..'\ud7a3' + | '\ud7b0'..'\ud7c6' + | '\ud7cb'..'\ud7fb' + | '\uf900'..'\ufa6d' + | '\ufa70'..'\ufad9' + | '\ufb1d'..'\ufb1f' + | '\ufb20'..'\ufb28' + | '\ufb2a'..'\ufb36' + | '\ufb38'..'\ufb3c' + | '\ufb3e'..'\ufb40' + | '\ufb41'..'\ufb43' + | '\ufb44'..'\ufb46' + | '\ufb47'..'\ufbb1' + | '\ufbd3'..'\ufd3d' + | '\ufd50'..'\ufd8f' + | '\ufd92'..'\ufdc7' + | '\ufdf0'..'\ufdfb' + | '\ufe70'..'\ufe74' + | '\ufe76'..'\ufefc' + | '\uff66'..'\uff6f' + | '\uff71'..'\uff9d' + | '\uffa0'..'\uffbe' + | '\uffc2'..'\uffc7' + | '\uffca'..'\uffcf' + | '\uffd2'..'\uffd7' + | '\uffda'..'\uffdc' + ; + +fragment UnicodeClassNL + : '\u16EE' // RUNIC ARLAUG SYMBOL + | '\u16EF' // RUNIC TVIMADUR SYMBOL + | '\u16F0' // RUNIC BELGTHOR SYMBOL + | '\u2160' // ROMAN NUMERAL ONE + | '\u2161' // ROMAN NUMERAL TWO + | '\u2162' // ROMAN NUMERAL THREE + | '\u2163' // ROMAN NUMERAL FOUR + | '\u2164' // ROMAN NUMERAL FIVE + | '\u2165' // ROMAN NUMERAL SIX + | '\u2166' // ROMAN NUMERAL SEVEN + | '\u2167' // ROMAN NUMERAL EIGHT + | '\u2168' // ROMAN NUMERAL NINE + | '\u2169' // ROMAN NUMERAL TEN + | '\u216A' // ROMAN NUMERAL ELEVEN + | '\u216B' // ROMAN NUMERAL TWELVE + | '\u216C' // ROMAN NUMERAL FIFTY + | '\u216D' // ROMAN NUMERAL ONE HUNDRED + | '\u216E' // ROMAN NUMERAL FIVE HUNDRED + | '\u216F' // ROMAN NUMERAL ONE THOUSAND + ; + +fragment UnicodeClassMN + : '\u0300' // COMBINING GRAVE ACCENT + | '\u0301' // COMBINING ACUTE ACCENT + | '\u0302' // COMBINING CIRCUMFLEX ACCENT + | '\u0303' // COMBINING TILDE + | '\u0304' // COMBINING MACRON + | '\u0305' // COMBINING OVERLINE + | '\u0306' // COMBINING BREVE + | '\u0307' // COMBINING DOT ABOVE + | '\u0308' // COMBINING DIAERESIS + | '\u0309' // COMBINING HOOK ABOVE + | '\u030A' // COMBINING RING ABOVE + | '\u030B' // COMBINING DOUBLE ACUTE ACCENT + | '\u030C' // COMBINING CARON + | '\u030D' // COMBINING VERTICAL LINE ABOVE + | '\u030E' // COMBINING DOUBLE VERTICAL LINE ABOVE + | '\u030F' // COMBINING DOUBLE GRAVE ACCENT + | '\u0310' // COMBINING CANDRABINDU + ; + +fragment UnicodeClassMC + : '\u0903' // DEVANAGARI SIGN VISARGA + | '\u093E' // DEVANAGARI VOWEL SIGN AA + | '\u093F' // DEVANAGARI VOWEL SIGN I + | '\u0940' // DEVANAGARI VOWEL SIGN II + | '\u0949' // DEVANAGARI VOWEL SIGN CANDRA O + | '\u094A' // DEVANAGARI VOWEL SIGN SHORT O + | '\u094B' // DEVANAGARI VOWEL SIGN O + | '\u094C' // DEVANAGARI VOWEL SIGN AU + ; + +fragment UnicodeClassCF + : '\u00AD' // SOFT HYPHEN + | '\u0600' // ARABIC NUMBER SIGN + | '\u0601' // ARABIC SIGN SANAH + | '\u0602' // ARABIC FOOTNOTE MARKER + | '\u0603' // ARABIC SIGN SAFHA + | '\u06DD' // ARABIC END OF AYAH + ; + +fragment UnicodeClassPC + : '\u005F' // LOW LINE + | '\u203F' // UNDERTIE + | '\u2040' // CHARACTER TIE + | '\u2054' // INVERTED UNDERTIE + | '\uFE33' // PRESENTATION FORM FOR VERTICAL LOW LINE + | '\uFE34' // PRESENTATION FORM FOR VERTICAL WAVY LOW LINE + | '\uFE4D' // DASHED LOW LINE + | '\uFE4E' // CENTRELINE LOW LINE + | '\uFE4F' // WAVY LOW LINE + | '\uFF3F' // FULLWIDTH LOW LINE + ; + +fragment UnicodeClassND + : '\u0030'..'\u0039' + | '\u0660'..'\u0669' + | '\u06f0'..'\u06f9' + | '\u07c0'..'\u07c9' + | '\u0966'..'\u096f' + | '\u09e6'..'\u09ef' + | '\u0a66'..'\u0a6f' + | '\u0ae6'..'\u0aef' + | '\u0b66'..'\u0b6f' + | '\u0be6'..'\u0bef' + | '\u0c66'..'\u0c6f' + | '\u0ce6'..'\u0cef' + | '\u0d66'..'\u0d6f' + | '\u0de6'..'\u0def' + | '\u0e50'..'\u0e59' + | '\u0ed0'..'\u0ed9' + | '\u0f20'..'\u0f29' + | '\u1040'..'\u1049' + | '\u1090'..'\u1099' + | '\u17e0'..'\u17e9' + | '\u1810'..'\u1819' + | '\u1946'..'\u194f' + | '\u19d0'..'\u19d9' + | '\u1a80'..'\u1a89' + | '\u1a90'..'\u1a99' + | '\u1b50'..'\u1b59' + | '\u1bb0'..'\u1bb9' + | '\u1c40'..'\u1c49' + | '\u1c50'..'\u1c59' + | '\ua620'..'\ua629' + | '\ua8d0'..'\ua8d9' + | '\ua900'..'\ua909' + | '\ua9d0'..'\ua9d9' + | '\ua9f0'..'\ua9f9' + | '\uaa50'..'\uaa59' + | '\uabf0'..'\uabf9' + | '\uff10'..'\uff19' + ; \ No newline at end of file diff --git a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java index c3e4f6e622..49229f3170 100644 --- a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java +++ b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java @@ -4,21 +4,19 @@ package net.sourceforge.pmd.cpd; -import java.io.BufferedReader; -import java.io.CharArrayReader; -import java.io.Closeable; -import java.io.IOException; -import java.io.PushbackReader; import java.util.Properties; -import org.apache.commons.lang3.RandomStringUtils; +import org.antlr.v4.runtime.CharStream; + +import net.sourceforge.pmd.cpd.token.AntlrToken; +import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; +import net.sourceforge.pmd.lang.antlr.AntlrTokenManager; +import net.sourceforge.pmd.lang.cs.antlr4.CSharpLexer; /** - * This class does a best-guess try-anything tokenization. - * - * @author jheintz + * The C# tokenizer. */ -public class CsTokenizer implements Tokenizer { +public class CsTokenizer extends AntlrTokenizer { private boolean ignoreUsings = false; @@ -28,283 +26,128 @@ public class CsTokenizer implements Tokenizer { } } - @Override - public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { - try (Tokenizer tokenizer = new Tokenizer(sourceCode.getCodeBuffer().toString())) { - Token token = tokenizer.getNextToken(); - - while (!token.equals(Token.EOF)) { - Token lookAhead = tokenizer.getNextToken(); - - // Ignore using directives - // Only using directives should be ignored, because these are used - // to import namespaces - // - // Using directive: 'using System.Math;' - // Using statement: 'using (Font font1 = new Font(..)) { .. }' - if (ignoreUsings && "using".equals(token.image) && !"(".equals(lookAhead.image)) { - // We replace the 'using' token by a random token, because it - // should not be part of - // any duplication block. When we omit it from the token stream, - // there is a change that - // we get a duplication block that starts before the 'using' - // directives and ends afterwards. - String randomTokenText = RandomStringUtils.randomAlphanumeric(20); - - token = new Token(randomTokenText, token.lineNumber); - // Skip all other tokens of the using directive to prevent a - // partial matching - while (!";".equals(lookAhead.image) && !lookAhead.equals(Token.EOF)) { - lookAhead = tokenizer.getNextToken(); - } - } - if (!";".equals(token.image)) { - tokenEntries.add(new TokenEntry(token.image, sourceCode.getFileName(), token.lineNumber)); - } - token = lookAhead; - } - tokenEntries.add(TokenEntry.getEOF()); - } catch (IOException e) { - e.printStackTrace(); - } - } - public void setIgnoreUsings(boolean ignoreUsings) { this.ignoreUsings = ignoreUsings; } - private static class Tokenizer implements Closeable { - private boolean endOfFile; - private int line; - private final PushbackReader reader; + @Override + protected AntlrTokenManager getLexerForSource(final SourceCode sourceCode) { + final CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); + return new AntlrTokenManager(new CSharpLexer(charStream), sourceCode.getFileName()); + } - Tokenizer(String sourceCode) { - endOfFile = false; - line = 1; - reader = new PushbackReader(new BufferedReader(new CharArrayReader(sourceCode.toCharArray()))); + @Override + protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) { + return new CsTokenFilter(tokenManager, ignoreUsings); + } + + /** + * The {@link CsTokenFilter} extends the {@link AntlrTokenFilter} to discard + * C#-specific tokens. + *

+ * By default, it enables annotation-based CPD suppression. + * If the --ignoreUsings flag is provided, using directives are filtered out. + *

+ */ + private static class CsTokenFilter extends AntlrTokenFilter { + private enum UsingState { + KEYWORD, // just encountered the using keyword + IDENTIFIER, // just encountered an identifier or var keyword } - public Token getNextToken() { - if (endOfFile) { - return Token.EOF; - } + private final boolean ignoreUsings; + private boolean discardingUsings = false; + private boolean discardingNL = false; - try { - int ic = reader.read(); - char c; - StringBuilder b; - while (ic != -1) { - c = (char) ic; - switch (c) { - // new line - case '\n': - line++; - ic = reader.read(); - break; - - // white space - case ' ': - case '\t': - case '\r': - ic = reader.read(); - break; - - case ';': - return new Token(";", line); - - // < << <= <<= > >> >= >>= - case '<': - case '>': - ic = reader.read(); - if (ic == '=') { - return new Token(c + "=", line); - } else if (ic == c) { - ic = reader.read(); - if (ic == '=') { - return new Token(c + c + "=", line); - } else { - reader.unread(ic); - return new Token(String.valueOf(c) + c, line); - } - } else { - reader.unread(ic); - return new Token(String.valueOf(c), line); - } - - // = == & &= && | |= || + += ++ - -= -- - case '=': - case '&': - case '|': - case '+': - case '-': - ic = reader.read(); - if (ic == '=' || ic == c) { - return new Token(c + String.valueOf((char) ic), line); - } else { - reader.unread(ic); - return new Token(String.valueOf(c), line); - } - - // ! != * *= % %= ^ ^= ~ ~= - case '!': - case '*': - case '%': - case '^': - case '~': - ic = reader.read(); - if (ic == '=') { - return new Token(c + "=", line); - } else { - reader.unread(ic); - return new Token(String.valueOf(c), line); - } - - // strings & chars - case '"': - case '\'': - int beginLine = line; - b = new StringBuilder(); - b.append(c); - while ((ic = reader.read()) != c) { - if (ic == -1) { - break; - } - b.append((char) ic); - if (ic == '\\') { - int next = reader.read(); - if (next != -1) { - b.append((char) next); - - if (next == '\n') { - line++; - } - } - } else if (ic == '\n') { - line++; - } - } - if (ic != -1) { - b.append((char) ic); - } - return new Token(b.toString(), beginLine); - - // / /= /*...*/ //... - case '/': - ic = reader.read(); - c = (char) ic; - switch (c) { - case '*': - // int beginLine = line; - int state = 1; - b = new StringBuilder(); - b.append("/*"); - - while ((ic = reader.read()) != -1) { - c = (char) ic; - b.append(c); - - if (c == '\n') { - line++; - } - - if (state == 1) { - if (c == '*') { - state = 2; - } - } else { - if (c == '/') { - ic = reader.read(); - break; - } else if (c != '*') { - state = 1; - } - } - } - // ignore the /* comment - // tokenEntries.add(new TokenEntry(b.toString(), - // sourceCode.getFileName(), beginLine)); - break; - - case '/': - b = new StringBuilder(); - b.append("//"); - while ((ic = reader.read()) != '\n') { - if (ic == -1) { - break; - } - b.append((char) ic); - } - // ignore the // comment - // tokenEntries.add(new TokenEntry(b.toString(), - // sourceCode.getFileName(), line)); - break; - - case '=': - return new Token("/=", line); - - default: - reader.unread(ic); - return new Token("/", line); - } - break; - - default: - // [a-zA-Z_][a-zA-Z_0-9]* - if (Character.isJavaIdentifierStart(c)) { - b = new StringBuilder(); - do { - b.append(c); - ic = reader.read(); - c = (char) ic; - } while (Character.isJavaIdentifierPart(c)); - reader.unread(ic); - return new Token(b.toString(), line); - } else if (Character.isDigit(c) || c == '.') { - // numbers - b = new StringBuilder(); - do { - b.append(c); - if (c == 'e' || c == 'E') { - ic = reader.read(); - c = (char) ic; - if ("1234567890-".indexOf(c) == -1) { - break; - } - b.append(c); - } - ic = reader.read(); - c = (char) ic; - } while ("1234567890.iIlLfFdDsSuUeExX".indexOf(c) != -1); - reader.unread(ic); - return new Token(b.toString(), line); - } else { - // anything else - return new Token(String.valueOf(c), line); - } - } - } - } catch (IOException e) { - e.printStackTrace(); - } - endOfFile = true; - return Token.EOF; + CsTokenFilter(final AntlrTokenManager tokenManager, boolean ignoreUsings) { + super(tokenManager); + this.ignoreUsings = ignoreUsings; } @Override - public void close() throws IOException { - reader.close(); + protected void analyzeToken(final AntlrToken currentToken) { + skipNewLines(currentToken); } - } - private static class Token { - public static final Token EOF = new Token("EOF", -1); + @Override + protected void analyzeTokens(final AntlrToken currentToken, final Iterable remainingTokens) { + skipUsingDirectives(currentToken, remainingTokens); + } - public final String image; - public final int lineNumber; + private void skipUsingDirectives(final AntlrToken currentToken, final Iterable remainingTokens) { + final int type = currentToken.getType(); + if (type == CSharpLexer.USING && isUsingDirective(remainingTokens)) { + discardingUsings = true; + } else if (type == CSharpLexer.SEMICOLON) { + discardingUsings = false; + } + } - Token(String image, int lineNumber) { - this.image = image; - this.lineNumber = lineNumber; + private boolean isUsingDirective(final Iterable remainingTokens) { + UsingState usingState = UsingState.KEYWORD; + for (final AntlrToken token : remainingTokens) { + final int type = token.getType(); + if (usingState == UsingState.KEYWORD) { + // The previous token was a using keyword. + switch (type) { + case CSharpLexer.STATIC: + // Definitely a using directive. + // Example: using static System.Math; + return true; + case CSharpLexer.VAR: + // Definitely a using statement. + // Example: using var font1 = new Font("Arial", 10.0f); + return false; + case CSharpLexer.OPEN_PARENS: + // Definitely a using statement. + // Example: using (var font1 = new Font("Arial", 10.0f); + return false; + case CSharpLexer.IDENTIFIER: + // This is either a type for a using statement or an alias for a using directive. + // Example (directive): using Project = PC.MyCompany.Project; + // Example (statement): using Font font1 = new Font("Arial", 10.0f); + usingState = UsingState.IDENTIFIER; + break; + default: + // Some unknown construct? + return false; + } + } else if (usingState == UsingState.IDENTIFIER) { + // The previous token was an identifier. + switch (type) { + case CSharpLexer.ASSIGNMENT: + // Definitely a using directive. + // Example: using Project = PC.MyCompany.Project; + return true; + case CSharpLexer.IDENTIFIER: + // Definitely a using statement. + // Example: using Font font1 = new Font("Arial", 10.0f); + return false; + case CSharpLexer.DOT: + // This should be considered part of the same type; revert to previous state. + // Example (directive): using System.Text; + // Example (statement): using System.Drawing.Font font1 = new Font("Arial", 10.0f); + usingState = UsingState.KEYWORD; + break; + case CSharpLexer.SEMICOLON: + // End of using directive. + return true; + default: + // Some unknown construct? + return false; + } + } + } + return false; + } + + private void skipNewLines(final AntlrToken currentToken) { + discardingNL = currentToken.getType() == CSharpLexer.NL; + } + + @Override + protected boolean isLanguageSpecificDiscarding() { + return discardingUsings || discardingNL; } } } diff --git a/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java b/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java index 20aefc8194..ed87562537 100644 --- a/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java +++ b/pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java @@ -12,6 +12,8 @@ import java.util.List; import org.junit.Before; import org.junit.Test; +import net.sourceforge.pmd.lang.ast.TokenMgrError; + public class CsTokenizerTest { private CsTokenizer tokenizer; @@ -46,7 +48,7 @@ public class CsTokenizerTest { public void testSimpleClassMethodMultipleLines() { tokenizer.tokenize(toSourceCode("class Foo {\n" + " public String foo(int a) {\n" + " int i = a;\n" + " return \"x\" + a;\n" + " }\n" + "}"), tokens); - assertEquals(22, tokens.size()); + assertEquals(24, tokens.size()); List tokenList = tokens.getTokens(); assertEquals(1, tokenList.get(0).getBeginLine()); assertEquals(2, tokenList.get(4).getBeginLine()); @@ -56,13 +58,12 @@ public class CsTokenizerTest { @Test public void testStrings() { tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens); - assertEquals(5, tokens.size()); + assertEquals(6, tokens.size()); } - @Test + @Test(expected = TokenMgrError.class) public void testOpenString() { tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens); - assertEquals(5, tokens.size()); } @Test @@ -91,7 +92,7 @@ public class CsTokenizerTest { + " a++; \n" + " a /= 3e2; \n" + " float f = -3.1; \n" + " f *= 2; \n" + " bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n" + " }\n" + "}"), tokens); - assertEquals(50, tokens.size()); + assertEquals(57, tokens.size()); } @Test @@ -119,8 +120,9 @@ public class CsTokenizerTest { public void testIgnoreUsingDirectives() { tokenizer.setIgnoreUsings(true); tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens); + assertEquals(1, tokens.size()); assertNotEquals("using", tokens.getTokens().get(0).toString()); - assertEquals(2, tokens.size()); + assertEquals(TokenEntry.EOF, tokens.getTokens().get(0)); } @Test @@ -132,6 +134,15 @@ public class CsTokenizerTest { assertEquals("using", tokens.getTokens().get(0).toString()); } + @Test + public void testUsingVarStatementsAreNotIgnored() { + tokenizer.setIgnoreUsings(true); + tokenizer.tokenize(toSourceCode( + "using var font1 = new Font(\"Arial\", 10.0f);\n" + " byte charset = font1.GdiCharSet;\n"), + tokens); + assertEquals("using", tokens.getTokens().get(0).toString()); + } + private SourceCode toSourceCode(String source) { return new SourceCode(new SourceCode.StringCodeLoader(source)); } From 4bd5a159e504b7f92e00e779da214c4ca102f0b8 Mon Sep 17 00:00:00 2001 From: Maikel Steneker Date: Thu, 27 Feb 2020 11:36:28 +0100 Subject: [PATCH 3/4] Rewrite to avoid Guava dependency. --- pmd-apex-jorje/pom.xml | 2 +- pmd-core/pom.xml | 6 -- .../cpd/token/internal/BaseTokenFilter.java | 13 +++-- .../pmd/internal/util/IteratorUtil.java | 56 +++++++++++++++++++ .../token/internal/BaseTokenFilterTest.java | 6 +- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index 52c7b7c313..ab7238f975 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -81,7 +81,7 @@ com.google.guava guava - 26.0-android + 26.0-jre com.google.j2objc diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 80d980a398..80aab40792 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -186,12 +186,6 @@ system-rules test - - com.google.guava - guava - 26.0-android - compile - diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java index 67cb75937c..7b7f11ca17 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilter.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.cpd.token.internal; +import static net.sourceforge.pmd.internal.util.IteratorUtil.AbstractIterator; + import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.LinkedList; @@ -12,8 +14,6 @@ import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; -import com.google.common.collect.AbstractIterator; - /** * A generic filter for PMD token managers that allows to use comments * to enable / disable analysis of parts of the stream @@ -136,21 +136,22 @@ public abstract class BaseTokenFilter implements TokenFi } @Override - protected T computeNext() { + protected void computeNext() { assert index >= 0; if (startToken != currentToken) { // NOPMD - intentional check for reference equality throw new ConcurrentModificationException("Using iterator after next token has been requested."); } if (index < unprocessedTokens.size()) { - return unprocessedTokens.get(index++); + setNext(unprocessedTokens.get(index++)); } else { final T nextToken = (T) tokenManager.getNextToken(); if (shouldStopProcessing(nextToken)) { - return endOfData(); + done(); + return; } index++; unprocessedTokens.add(nextToken); - return nextToken; + setNext(nextToken); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/IteratorUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/IteratorUtil.java index b1d323cebe..e676447aef 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/IteratorUtil.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/IteratorUtil.java @@ -9,6 +9,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.NoSuchElementException; /** @@ -88,4 +89,59 @@ public final class IteratorUtil { } }; } + + public abstract static class AbstractIterator implements Iterator { + + private State state = State.NOT_READY; + private T next = null; + + + @Override + public boolean hasNext() { + switch (state) { + case DONE: + return false; + case READY: + return true; + default: + state = null; + computeNext(); + if (state == null) { + throw new IllegalStateException("Should have called done or setNext"); + } + return state == State.READY; + } + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + state = State.NOT_READY; + return next; + } + + protected final void setNext(T t) { + next = t; + state = State.READY; + } + + protected final void done() { + state = State.DONE; + } + + protected abstract void computeNext(); + + enum State { + READY, NOT_READY, DONE + } + + @Deprecated + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } + + } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java index 21da8ccc19..2478e0c913 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java @@ -8,6 +8,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.NoSuchElementException; @@ -17,8 +19,6 @@ import org.junit.Test; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.GenericToken; -import com.google.common.collect.ImmutableList; - public class BaseTokenFilterTest { class StringToken implements GenericToken { @@ -67,7 +67,7 @@ public class BaseTokenFilterTest { class StringTokenManager implements TokenManager { - Iterator iterator = ImmutableList.of("a", "b", "c").iterator(); + Iterator iterator = Collections.unmodifiableList(Arrays.asList("a", "b", "c")).iterator(); @Override public Object getNextToken() { From b65a6a52eace051d05fa7b99c4b33a773c5c56de Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 29 Feb 2020 11:34:50 +0100 Subject: [PATCH 4/4] [doc] Update release notes, fixes #2139, refs #2280. --- docs/pages/release_notes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index ed2b154b84..01b00b90f9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -19,8 +19,16 @@ This is a {{ site.pmd.release_type }} release. This PMD release ships a new version of the pmd-designer. For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designer/releases/tag/6.21.0). +#### Improved CPD support for C# + +The C# tokenizer is now based on an antlr grammar instead of a manual written tokenizer. This +should give more accurate results and especially fixes the problems with the using statement syntax +(see [#2139](https://github.com/pmd/pmd/issues/2139)). + ### Fixed Issues +* cs + * [#2139](https://github.com/pmd/pmd/issues/2139): \[cs] CPD doesn't understand alternate using statement syntax with C# 8.0 * java-errorprone * [#2250](https://github.com/pmd/pmd/issues/2250): \[java] InvalidLogMessageFormat flags logging calls using a slf4j-Marker @@ -31,6 +39,7 @@ For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designe * [#2251](https://github.com/pmd/pmd/pull/2251): \[java] FP for InvalidLogMessageFormat when using slf4j-Markers - [Kris Scheibe](https://github.com/kris-scheibe) * [#2253](https://github.com/pmd/pmd/pull/2253): \[modelica] Remove duplicated dependencies - [Piotrek Żygieło](https://github.com/pzygielo) * [#2256](https://github.com/pmd/pmd/pull/2256): \[doc] Corrected XML attributes in release notes - [Maikel Steneker](https://github.com/maikelsteneker) +* [#2280](https://github.com/pmd/pmd/pull/2280): \[cs] CPD: Replace C# tokenizer by an Antlr-based one - [Maikel Steneker](https://github.com/maikelsteneker) {% endtocmaker %}