From 315bb87c7f6d0aee1869070cf3d2eb32d671ef10 Mon Sep 17 00:00:00 2001
From: kenji21 <r.bergoin@openium.fr>
Date: Thu, 2 Nov 2017 13:50:10 +0100
Subject: [PATCH 01/53] Update Swift.g4 grammar to have swift4 keypath support

---
 .../pmd/lang/swift/antlr4/Swift.g4            | 434 +++++++++++-------
 .../pmd/cpd/SwiftTokenizerTest.java           |   2 +-
 2 files changed, 262 insertions(+), 174 deletions(-)

diff --git a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4 b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
index 3a8fef8af5..605800b350 100644
--- a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
+++ b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
@@ -33,7 +33,13 @@
  */
 grammar Swift;
 
-topLevel : (statement | expression)* EOF ;
+/*
+@header {
+package com.sleekbyte.tailor.antlr;
+}
+*/
+
+topLevel : statements? EOF ;
 
 // Statements
 
@@ -55,33 +61,22 @@ statements : statement+ ;
 
 // GRAMMAR OF A LOOP STATEMENT
 
-loopStatement : forStatement
- | forInStatement
+loopStatement : forInStatement
  | whileStatement
  | repeatWhileStatement
  ;
 
-// GRAMMAR OF A FOR STATEMENT
-
-// Swift Language Reference has expression? instead of expressionList?
-forStatement
- : 'for' forInit? ';' expression? ';' expressionList? codeBlock
- | 'for' '(' forInit?';' expression? ';' expressionList? ')' codeBlock
- ;
-
-forInit : variableDeclaration | expressionList  ;
-
 // GRAMMAR OF A FOR_IN STATEMENT
 
 forInStatement : 'for' 'case'? pattern 'in' expression whereClause? codeBlock  ;
 
 // GRAMMAR OF A WHILE STATEMENT
 
-whileStatement : 'while' conditionClause codeBlock  ;
+whileStatement : 'while' conditionList codeBlock  ;
 
 // GRAMMAR OF A REPEAT WHILE STATEMENT
 
-repeatWhileStatement: 'repeat' codeBlock 'while' conditionClause ;
+repeatWhileStatement: 'repeat' codeBlock 'while' expression ;
 
 // GRAMMAR OF A BRANCH STATEMENT
 
@@ -89,17 +84,17 @@ branchStatement : ifStatement | guardStatement | switchStatement  ;
 
 // GRAMMAR OF AN IF STATEMENT
 
-ifStatement : 'if' conditionClause codeBlock elseClause? ;
+ifStatement : 'if' conditionList codeBlock elseClause? ;
 elseClause : 'else' codeBlock | 'else' ifStatement  ;
 
 // GRAMMAR OF A GUARD STATEMENT
 
-guardStatement : 'guard' conditionClause 'else' codeBlock ;
+guardStatement : 'guard' conditionList 'else' codeBlock ;
 
 // GRAMMAR OF A SWITCH STATEMENT
 
 switchStatement : 'switch' expression '{' switchCases? '}'  ;
-switchCases : switchCase switchCases? ;
+switchCases : switchCase+ ;
 switchCase : caseLabel statements | defaultLabel statements  | caseLabel ';' | defaultLabel ';'  ;
 caseLabel : 'case' caseItemList ':' ;
 caseItemList : caseItem (',' caseItem)* ;
@@ -108,7 +103,7 @@ defaultLabel : 'default' ':' ;
 
 // GRAMMAR OF A LABELED STATEMENT
 
-labeledStatement : statementLabel loopStatement | statementLabel switchStatement  ;
+labeledStatement : statementLabel (loopStatement | ifStatement | switchStatement | doStatement)  ;
 statementLabel : labelName ':' ;
 labelName : identifier  ;
 
@@ -151,22 +146,10 @@ doStatement: 'do' codeBlock catchClauses? ;
 catchClauses: catchClause catchClauses? ;
 catchClause: 'catch' pattern? whereClause? codeBlock ;
 
-// GRAMMAR FOR CONDITION CLAUSES
-
-conditionClause : expression
- | expression ',' conditionList
- | conditionList
- | availabilityCondition ',' expression
- ;
-
 conditionList : condition (',' condition)* ;
-condition: availabilityCondition | caseCondition | optionalBindingCondition ;
-caseCondition: 'case' pattern initializer whereClause? ;
-// optionalBindingCondition is incorrect in the Swift Language Reference (missing a ',')
-optionalBindingCondition: optionalBindingHead (',' optionalBindingContinuationList)? whereClause? ;
-optionalBindingHead: 'let' pattern initializer | 'var' pattern initializer ;
-optionalBindingContinuationList: optionalBindingContinuation (',' optionalBindingContinuation)* ;
-optionalBindingContinuation: optionalBindingHead | pattern initializer ;
+condition: availabilityCondition | caseCondition | optionalBindingCondition | expression ;
+caseCondition: 'case' pattern initializer ;
+optionalBindingCondition: ('let'|'var') pattern initializer ;
 
 whereClause: 'where' whereExpression ;
 whereExpression: expression ;
@@ -184,10 +167,10 @@ platformVersion: VersionLiteral | DecimalLiteral | FloatingPointLiteral ; // TOD
 
 // GRAMMAR OF A GENERIC PARAMETER CLAUSE
 
-genericParameterClause : '<' genericParameterList requirementClause? '>'  ;
+genericParameterClause : '<' genericParameterList '>'  ;
 genericParameterList : genericParameter (',' genericParameter)*  ;
 genericParameter : typeName | typeName ':' typeIdentifier | typeName ':' protocolCompositionType  ;
-requirementClause : 'where' requirementList  ;
+genericWhereClause : 'where' requirementList  ;
 requirementList : requirement (',' requirement)*  ;
 requirement : conformanceRequirement | sameTypeRequirement  ;
 conformanceRequirement : typeIdentifier ':' typeIdentifier | typeIdentifier ':' protocolCompositionType  ;
@@ -220,20 +203,26 @@ declaration
  | operatorDeclaration ';'?
  // compiler-control-statement not in Swift Language Reference
  | compilerControlStatement ';'?
+ | precedenceGroupDeclaration ';'?
  ;
 
-declarations : declaration declarations? ;
-declarationModifiers : declarationModifier declarationModifiers? ;
+declarations : declaration+ ;
+declarationModifiers : declarationModifier+ ;
 declarationModifier : 'class' | 'convenience' | 'dynamic' | 'final' | 'infix'
- | 'lazy' | 'mutating' | 'nonmutating' | 'optional' | 'override' | 'postfix'
+ | 'lazy' | 'optional' | 'override' | 'postfix'
  | 'prefix' | 'required' | 'static' | 'unowned' | 'unowned' '(' 'safe' ')'
  | 'unowned' '(' 'unsafe' ')' | 'weak'
- | accessLevelModifier ;
+ | accessLevelModifier
+ | mutationModifier ;
 
-accessLevelModifier : 'internal' | 'internal' '(' 'set' ')'
- | 'private' | 'private' '(' 'set' ')'
- | 'public' | 'public' '(' 'set' ')' ;
-accessLevelModifiers : accessLevelModifier accessLevelModifiers? ;
+accessLevelModifier : 'private' | 'private' '(' 'set' ')'
+ | 'fileprivate' | 'fileprivate' '(' 'set' ')'
+ | 'internal' | 'internal' '(' 'set' ')'
+ | 'public' | 'public' '(' 'set' ')'
+ | 'open' | 'open' '(' 'set' ')' ;
+accessLevelModifiers : accessLevelModifier+ ;
+
+mutationModifier: 'mutating' | 'nonmutating' ;
 
 // GRAMMAR OF A CODE BLOCK
 
@@ -242,7 +231,8 @@ codeBlock : '{' statements? '}'  ;
 // GRAMMAR OF AN IMPORT DECLARATION
 
 importDeclaration : attributes? 'import' importKind? importPath  ;
-importKind : 'typealias' | 'struct' | 'class' | 'enum' | 'protocol' | 'var' | 'func'  ;
+// Swift Language Reference does not have let
+importKind : 'typealias' | 'struct' | 'class' | 'enum' | 'protocol' | 'var' | 'func' | 'let'  ;
 importPath : importPathIdentifier | importPathIdentifier '.' importPath  ;
 importPathIdentifier : identifier | operator  ;
 
@@ -268,8 +258,10 @@ variableDeclaration
 variableDeclarationHead : attributes? declarationModifiers? 'var'  ;
 variableName : identifier  ;
 getterSetterBlock : '{' getterClause setterClause?'}'  | '{' setterClause getterClause '}'  ;
-getterClause : attributes? 'get' codeBlock  ;
-setterClause : attributes? 'set' setterName? codeBlock  ;
+// declarationModifiers missing in the Swift Language Reference
+getterClause : attributes? declarationModifiers? 'get' codeBlock  ;
+// declarationModifiers missing in the Swift Language Reference
+setterClause : attributes? declarationModifiers? 'set' setterName? codeBlock  ;
 setterName : '(' identifier ')'  ;
 getterSetterKeywordBlock : '{' getterKeywordClause setterKeywordClause?'}' | '{' setterKeywordClause getterKeywordClause '}'  ;
 getterKeywordClause : attributes? 'get'  ;
@@ -281,7 +273,7 @@ didSetClause : attributes? 'didSet' setterName? codeBlock  ;
 // GRAMMAR OF A TYPE ALIAS DECLARATION
 
 typealiasDeclaration : typealiasHead typealiasAssignment  ;
-typealiasHead : attributes? accessLevelModifier? 'typealias' typealiasName  ;
+typealiasHead : attributes? accessLevelModifier? 'typealias' typealiasName genericParameterClause?  ;
 typealiasName : identifier  ;
 typealiasAssignment : '=' sType  ;
 
@@ -290,45 +282,44 @@ typealiasAssignment : '=' sType  ;
 /* HACK: functionBody? is intentionally not used to force the parser to try and match a functionBody first
  * This can be removed once we figure out how to enforce that statements are either separated by semi colons or new line characters
  */
-functionDeclaration : functionHead functionName genericParameterClause? functionSignature functionBody
- | functionHead functionName genericParameterClause? functionSignature
+functionDeclaration : functionHead functionName genericParameterClause? functionSignature genericWhereClause? functionBody
+ | functionHead functionName genericParameterClause? functionSignature genericWhereClause?
  ;
 
 functionHead : attributes? declarationModifiers? 'func'  ;
 functionName : identifier |  operator  ;
 // rethrows is not marked as optional in the Swift Language Reference
-functionSignature : parameterClauses ('throws' | 'rethrows')? functionResult? ;
+functionSignature : parameterClause ('throws' | 'rethrows')? functionResult? ;
 functionResult : '->' attributes? sType  ;
 functionBody : codeBlock  ;
-parameterClauses : parameterClause parameterClauses? ;
 parameterClause : '(' ')' |  '(' parameterList '...'? ')'  ;
 parameterList : parameter (',' parameter)*  ;
 // Parameters don't have attributes in the Swift Language Reference
-parameter : attributes? 'inout'? 'let'? '#'? externalParameterName? localParameterName typeAnnotation? defaultArgumentClause?
- | 'inout'? 'var' '#'? externalParameterName? localParameterName typeAnnotation? defaultArgumentClause?
- | attributes? sType
- | externalParameterName? localParameterName typeAnnotation '...'
+parameter
+ : attributes? externalParameterName? localParameterName typeAnnotation defaultArgumentClause?
+ | attributes? externalParameterName? localParameterName typeAnnotation '...'
  ;
-externalParameterName : identifier | '_'  ;
-localParameterName : identifier | '_'  ;
+// Swift Language Reference does not have "keyword" or "_"
+externalParameterName : identifier | keyword | '_';
+localParameterName : identifier | '_' ;
 defaultArgumentClause : '=' expression  ;
 
 // GRAMMAR OF AN ENUMERATION DECLARATION
 
 enumDeclaration : attributes? accessLevelModifier? enumDef  ;
 enumDef: unionStyleEnum | rawValueStyleEnum  ;
-unionStyleEnum : 'indirect'? 'enum' enumName genericParameterClause? typeInheritanceClause? '{' unionStyleEnumMembers?'}'  ;
-unionStyleEnumMembers : unionStyleEnumMember unionStyleEnumMembers? ;
-unionStyleEnumMember : declaration | unionStyleEnumCaseClause ';'? ;
+unionStyleEnum : 'indirect'? 'enum' enumName genericParameterClause? typeInheritanceClause? genericWhereClause? '{' unionStyleEnumMembers?'}'  ;
+unionStyleEnumMembers : unionStyleEnumMember+ ;
+unionStyleEnumMember : declaration | unionStyleEnumCaseClause ';'? | compilerControlStatement ;
 unionStyleEnumCaseClause : attributes? 'indirect'? 'case' unionStyleEnumCaseList  ;
 unionStyleEnumCaseList : unionStyleEnumCase (',' unionStyleEnumCase)*  ;
 unionStyleEnumCase : enumCaseName tupleType? ;
 enumName : identifier  ;
 enumCaseName : identifier  ;
 // typeInheritanceClause is not optional in the Swift Language Reference
-rawValueStyleEnum : 'enum' enumName genericParameterClause? typeInheritanceClause? '{' rawValueStyleEnumMembers?'}'  ;
-rawValueStyleEnumMembers : rawValueStyleEnumMember rawValueStyleEnumMembers? ;
-rawValueStyleEnumMember : declaration | rawValueStyleEnumCaseClause  ;
+rawValueStyleEnum : 'enum' enumName genericParameterClause? typeInheritanceClause? genericWhereClause? '{' rawValueStyleEnumMembers?'}'  ;
+rawValueStyleEnumMembers : rawValueStyleEnumMember+ ;
+rawValueStyleEnumMember : declaration | rawValueStyleEnumCaseClause | compilerControlStatement  ;
 rawValueStyleEnumCaseClause : attributes? 'case' rawValueStyleEnumCaseList  ;
 rawValueStyleEnumCaseList : rawValueStyleEnumCase (',' rawValueStyleEnumCase)*   ;
 rawValueStyleEnumCase : enumCaseName rawValueAssignment? ;
@@ -336,29 +327,34 @@ rawValueAssignment : '=' literal  ;
 
 // GRAMMAR OF A STRUCTURE DECLARATION
 
-structDeclaration : attributes? accessLevelModifier? 'struct' structName genericParameterClause? typeInheritanceClause? structBody  ;
+structDeclaration : attributes? accessLevelModifier? 'struct' structName genericParameterClause? typeInheritanceClause? genericWhereClause? structBody  ;
 structName : identifier  ;
-structBody : '{' declarations?'}'  ;
+structBody : '{' structMembers? '}'  ;
+structMembers: structMember+ ;
+structMember: declaration | compilerControlStatement ;
 
 // GRAMMAR OF A CLASS DECLARATION
 
-// declarationModifier missing in Swift Language Reference
-classDeclaration : attributes? declarationModifier* 'class' className genericParameterClause? typeInheritanceClause? classBody  ;
+classDeclaration : attributes? classDeclarationModifiers? 'class' className genericParameterClause? typeInheritanceClause? genericWhereClause? classBody  ;
+classDeclarationModifiers: accessLevelModifier 'final'? | 'final' accessLevelModifier? ;
 className : identifier ;
-classBody : '{' declarations? '}'  ;
+classBody : '{' classMembers? '}'  ;
+classMembers: classMember+ ;
+classMember: declaration | compilerControlStatement ;
 
 // GRAMMAR OF A PROTOCOL DECLARATION
 
 protocolDeclaration : attributes? accessLevelModifier? 'protocol' protocolName typeInheritanceClause? protocolBody  ;
 protocolName : identifier  ;
-protocolBody : '{' protocolMemberDeclarations?'}'  ;
+protocolBody : '{' protocolMembers? '}'  ;
+protocolMembers: protocolMember+ ;
+protocolMember: protocolMemberDeclaration | compilerControlStatement ;
 protocolMemberDeclaration : protocolPropertyDeclaration ';'?
  | protocolMethodDeclaration ';'?
  | protocolInitializerDeclaration ';'?
  | protocolSubscriptDeclaration ';'?
  | protocolAssociatedTypeDeclaration ';'?
  ;
-protocolMemberDeclarations : protocolMemberDeclaration protocolMemberDeclarations? ;
 
 // GRAMMAR OF A PROTOCOL PROPERTY DECLARATION
 
@@ -366,11 +362,11 @@ protocolPropertyDeclaration : variableDeclarationHead variableName typeAnnotatio
 
 // GRAMMAR OF A PROTOCOL METHOD DECLARATION
 
-protocolMethodDeclaration : functionHead functionName genericParameterClause? functionSignature  ;
+protocolMethodDeclaration : functionHead functionName genericParameterClause? functionSignature genericWhereClause?  ;
 
 // GRAMMAR OF A PROTOCOL INITIALIZER DECLARATION
 
-protocolInitializerDeclaration : initializerHead genericParameterClause? parameterClause ('throws' | 'rethrows')? ;
+protocolInitializerDeclaration : initializerHead genericParameterClause? parameterClause ('throws' | 'rethrows')? genericWhereClause? ;
 
 // GRAMMAR OF A PROTOCOL SUBSCRIPT DECLARATION
 
@@ -378,11 +374,11 @@ protocolSubscriptDeclaration : subscriptHead subscriptResult getterSetterKeyword
 
 // GRAMMAR OF A PROTOCOL ASSOCIATED TYPE DECLARATION
 
-protocolAssociatedTypeDeclaration : typealiasHead typeInheritanceClause? typealiasAssignment? ;
+protocolAssociatedTypeDeclaration : attributes? accessLevelModifier? 'associatedtype' typealiasName typeInheritanceClause? typealiasAssignment?;
 
 // GRAMMAR OF AN INITIALIZER DECLARATION
 
-initializerDeclaration : initializerHead genericParameterClause? parameterClause ('throws' | 'rethrows')? initializerBody  ;
+initializerDeclaration : initializerHead genericParameterClause? parameterClause ('throws' | 'rethrows')? genericWhereClause? initializerBody  ;
 initializerHead : attributes? declarationModifiers? 'init' ('?' | '!')?  ;
 initializerBody : codeBlock  ;
 
@@ -392,9 +388,10 @@ deinitializerDeclaration : attributes? 'deinit' codeBlock  ;
 
 // GRAMMAR OF AN EXTENSION DECLARATION
 
-// attributes, requirementClause missing in the Swift Language Reference
-extensionDeclaration : attributes? accessLevelModifier? 'extension' typeIdentifier requirementClause? typeInheritanceClause? extensionBody  ;
-extensionBody : '{' declarations?'}'  ;
+extensionDeclaration : attributes? accessLevelModifier? 'extension' typeIdentifier (genericWhereClause | typeInheritanceClause)? extensionBody  ;
+extensionBody : '{' extensionMembers?'}'  ;
+extensionMembers: extensionMember+ ;
+extensionMember: declaration | compilerControlStatement ;
 
 // GRAMMAR OF A SUBSCRIPT DECLARATION
 
@@ -409,15 +406,25 @@ subscriptResult : '->' attributes? sType  ;
 // GRAMMAR OF AN OPERATOR DECLARATION
 
 operatorDeclaration : prefixOperatorDeclaration | postfixOperatorDeclaration | infixOperatorDeclaration  ;
-prefixOperatorDeclaration : 'prefix' 'operator' operator '{' '}'  ;
-postfixOperatorDeclaration : 'postfix' 'operator' operator '{' '}'  ;
-infixOperatorDeclaration : 'infix' 'operator' operator '{' infixOperatorAttributes '}'  ;
-// Order of clauses can be reversed, not indicated in Swift Language Reference
-infixOperatorAttributes : precedenceClause? associativityClause? | associativityClause? precedenceClause? ;
-precedenceClause : 'precedence' precedenceLevel  ;
-precedenceLevel : integerLiteral ;
-associativityClause : 'associativity' associativity  ;
-associativity : 'left' | 'right' | 'none'  ;
+prefixOperatorDeclaration : 'prefix' 'operator' operator  ;
+postfixOperatorDeclaration : 'postfix' 'operator' operator  ;
+infixOperatorDeclaration : 'infix' 'operator' operator infixOperatorGroup? ;
+infixOperatorGroup: ':' precedenceGroupName ;
+
+
+// GRAMMAR OF A PRECEDENCE GROUP DECLARATION
+
+precedenceGroupDeclaration: 'precedencegroup' precedenceGroupName '{' precedenceGroupAttributes? '}' ;
+
+precedenceGroupAttributes: precedenceGroupAttribute+;
+precedenceGroupAttribute: precedenceGroupRelation | precedenceGroupAssignment | precedenceGroupAssociativity ;
+precedenceGroupRelation: ('higherThan' | 'lowerThan') ':' precedenceGroupNames;
+precedenceGroupAssignment: 'assignment' ':' booleanLiteral ;
+precedenceGroupAssociativity: 'associativity' ':' ('left' | 'right' | 'none') ;
+
+precedenceGroupNames: precedenceGroupName (',' precedenceGroupName)* ;
+precedenceGroupName: identifier ;
+
 
 // Patterns
 
@@ -450,10 +457,8 @@ valueBindingPattern : 'var' pattern | 'let' pattern  ;
 // GRAMMAR OF A TUPLE PATTERN
 
 tuplePattern : '(' tuplePatternElementList? ')'  ;
-tuplePatternElementList
-	:	tuplePatternElement (',' tuplePatternElement)*
-	;
-tuplePatternElement : pattern  ;
+tuplePatternElementList : tuplePatternElement (',' tuplePatternElement)* ;
+tuplePatternElement : pattern | identifier ':' pattern ;
 
 // GRAMMAR OF AN ENUMERATION CASE PATTERN
 
@@ -479,12 +484,13 @@ attributeName : identifier ;
 attributeArgumentClause : '('  balancedTokens?  ')'  ;
 attributes : attribute+ ;
 // Swift Language Reference does not have ','
-balancedTokens : balancedToken balancedTokens? ;
+balancedTokens : balancedToken+ ;
 balancedToken
  : '('  balancedTokens? ')'
  | '[' balancedTokens? ']'
  | '{' balancedTokens? '}'
- | identifier | expression | contextSensitiveKeyword | literal | operator
+ // VersionLiteral and availabilityArgument are not in the Swift Language Reference
+ | identifier | expression | contextSensitiveKeyword | literal | operator | VersionLiteral | availabilityArgument
  // | Any punctuation except ( ,  ')' , '[' , ']' , { , or }
  // Punctuation is very ambiguous, interpreting punctuation as defined in www.thepunctuationguide.com
  | ':' | ';' | ',' | '!' | '<' | '>' | '-' | '\'' | '/' | '...' | '"'
@@ -494,8 +500,6 @@ balancedToken
 
 // GRAMMAR OF AN EXPRESSION
 
-expressionList : expression (',' expression)* ;
-
 expression : tryOperator? prefixExpression binaryExpression* ;
 
 prefixExpression
@@ -505,24 +509,24 @@ prefixExpression
 
 /*
 expression
-	:	prefixOperator expression
-    |	inOutExpression
-    |	primaryExpression
-    |	expression binaryOperator expression
-    |	expression assignmentOperator expression
-    |	expression conditionalOperator expression
-    |	expression typeCastingOperator
-    |	expression postfixOperator
-    |	expression parenthesizedExpression trailingClosure?
-	|	expression '.' 'init'
- 	|	expression '.' DecimalLiteral
-	|	expression '.' identifier genericArgumentClause?
-	|	expression '.' 'self'
-	|	expression '.' 'dynamicType'
-	|	expression '[' expressionList ']'
-	|	expression '!'
-	|	expression '?'
-	;
+ : prefixOperator expression
+ | inOutExpression
+ | primaryExpression
+ | expression binaryOperator expression
+ | expression assignmentOperator expression
+ | expression conditionalOperator expression
+ | expression typeCastingOperator
+ | expression postfixOperator
+ | expression parenthesizedExpression trailingClosure?
+ | expression '.' 'init'
+ | expression '.' DecimalLiteral
+ | expression '.' identifier genericArgumentClause?
+ | expression '.' 'self'
+ | expression '.' 'dynamicType'
+ | expression '[' expressionList ']'
+ | expression '!'
+ | expression '?'
+ ;
 */
 
 // GRAMMAR OF A PREFIX EXPRESSION
@@ -562,15 +566,18 @@ typeCastingOperator
 // GRAMMAR OF A PRIMARY EXPRESSION
 
 primaryExpression
- : (identifier | operator) genericArgumentClause? // operator not mentioned in the Swift Language Reference
+ : (identifier | operator | keyword | contextSensitiveKeyword) genericArgumentClause? // operator, keyword, and contextSensitiveKeyword are not mentioned in the Swift Language Reference
  | literalExpression
  | selfExpression
  | superclassExpression
  | closureExpression
  | parenthesizedExpression
+ | tupleExpression
  | implicitMemberExpression
 // | implicit_member_expression disallow as ambig with explicit member expr in postfix_expression
  | wildcardExpression
+ | selectorExpression
+ | keyPathExpression
  ;
 
 // GRAMMAR OF A LITERAL EXPRESSION
@@ -579,23 +586,30 @@ literalExpression
  : literal
  | arrayLiteral
  | dictionaryLiteral
- | '__FILE__' | '__LINE__' | '__COLUMN__' | '__FUNCTION__'
+ | playgroundLiteral
+ | '#file' | '#line' | '#column' | '#function'
  ;
 
 arrayLiteral : '[' arrayLiteralItems? ']'  ;
 arrayLiteralItems : arrayLiteralItem (',' arrayLiteralItem)* ','?  ;
 arrayLiteralItem : expression ;
+
 dictionaryLiteral : '[' dictionaryLiteralItems ']' | '[' ':' ']'  ;
 dictionaryLiteralItems : dictionaryLiteralItem (',' dictionaryLiteralItem)* ','? ;
 dictionaryLiteralItem : expression ':' expression  ;
 
+playgroundLiteral: '#colorLiteral' '(' 'red' ':' expression ',' 'green' ':' expression ',' 'blue' ':' expression ',' 'alpha' ':' expression ')'
+ | '#fileLiteral' '(' 'resourceName' ':' expression ')'
+ | '#imageLiteral' '(' 'resourceName' ':' expression ')' ;
+
 // GRAMMAR OF A SELF EXPRESSION
 
 selfExpression
  : 'self'
- | 'self' '.' identifier
- | 'self' '[' expressionList ']'
- | 'self' '.' 'init'
+ | 'self' '.' identifier  // self-method-expression
+ // Swift Language Reference uses expressionList
+ | 'self' '[' tupleElementList ']'  // self-subscript-expression
+ | 'self' '.' 'init' // self-initializer-expression
  ;
 
 // GRAMMAR OF A SUPERCLASS EXPRESSION
@@ -607,23 +621,29 @@ superclassExpression
   ;
 
 superclassMethodExpression : 'super' '.' identifier  ;
-superclassSubscriptExpression : 'super' '[' expressionList ']'  ;
+// Swift Language Reference uses expressionList
+superclassSubscriptExpression : 'super' '[' tupleElementList ']'  ;
 superclassInitializerExpression : 'super' '.' 'init'  ;
 
 // GRAMMAR OF A CLOSURE EXPRESSION
 
-// Statements are not optional in the Swift Language Reference
 closureExpression : '{' closureSignature? statements? '}'  ;
+
 closureSignature
- : parameterClause functionResult? 'in'
- | identifierList functionResult? 'in'
- | captureList parameterClause functionResult? 'in'
- | captureList identifierList functionResult? 'in'
+ : captureList? closureParameterClause 'throws'? functionResult? 'in'
  | captureList 'in'
  ;
 
+closureParameterClause: '(' ')' | '(' closureParameterList ')' | identifierList ;
+closureParameterList: closureParameter (',' closureParameterList)* ;
+closureParameter: closureParameterName typeAnnotation?
+ | closureParameterName typeAnnotation '...'
+ ;
+// Swift Language Reference does not have "_"
+closureParameterName: identifier | '_';
+
 captureList : '[' captureListItems ']' ;
-captureListItems: captureListItem (',' captureListItem)? ;
+captureListItems: captureListItem (',' captureListItem)* ;
 captureListItem: captureSpecifier? expression ;
 captureSpecifier : 'weak' | 'unowned' | 'unowned(safe)' | 'unowned(unsafe)'  ;
 
@@ -633,51 +653,78 @@ implicitMemberExpression : '.' identifier  ;
 
 // GRAMMAR OF A PARENTHESIZED EXPRESSION
 
-parenthesizedExpression : '(' expressionElementList? ')'  ;
-expressionElementList : expressionElement (',' expressionElement)*  ;
-expressionElement : expression | identifier ':' expression  ;
+parenthesizedExpression : '(' expression ')'  ;
+
+// GRAMMAR OF A TUPLE EXPRESSION
+
+tupleExpression: '(' tupleElementList? ')' ;
+tupleElementList: tupleElement (',' tupleElement)* ;
+tupleElement: (identifier ':')? expression ;
 
 // GRAMMAR OF A WILDCARD EXPRESSION
 
 wildcardExpression : '_'  ;
 
+// GRAMMAR OF A SELECTOR EXPRESSION
+
+selectorExpression
+ : '#selector' '(' expression ')'
+ | '#selector' '(' ('getter:' | 'setter:') expression ')'
+ ;
+
+// GRAMMAR OF A KEY PATH EXPRESSION
+
+keyPathExpression
+ : '#keyPath' '(' expression ')' 
+ | '\\' expression
+;
+
 // GRAMMAR OF A POSTFIX EXPRESSION
 
 postfixExpression
  : primaryExpression                                             # primary
  | postfixExpression postfixOperator                             # postfixOperation
  // Function call with closure expression should always be above a lone parenthesized expression to reduce ambiguity
- | postfixExpression parenthesizedExpression? closureExpression  # functionCallWithClosureExpression
- | postfixExpression parenthesizedExpression                     # functionCallExpression
+ | postfixExpression functionCallArgumentClause? closureExpression  # functionCallWithClosureExpression
+ | postfixExpression functionCallArgumentClause                     # functionCallExpression
  | postfixExpression '.' 'init'                                  # initializerExpression
+ | postfixExpression '.' 'init' '(' argumentNames ')'            # initializerExpressionWithArguments
  // TODO: don't allow '_' here in DecimalLiteral:
  | postfixExpression '.' DecimalLiteral                         # explicitMemberExpression1
- | postfixExpression '.' identifier genericArgumentClause?     # explicitMemberExpression2
+ | postfixExpression '.' identifier genericArgumentClause?      # explicitMemberExpression2
+ | postfixExpression '.' identifier '(' argumentNames ')'       # explicitMemberExpression3
  | postfixExpression '.' 'self'                                  # postfixSelfExpression
- | postfixExpression '.' 'dynamicType'                           # dynamicTypeExpression
- | postfixExpression '[' expressionList ']'                     # subscriptExpression
+ | 'type' '(' 'of' ':' expression ')'                            # dynamicTypeExpression
+ // Swift Language Reference uses expressionList
+ | postfixExpression '[' tupleElementList ']'                     # subscriptExpression
  | postfixExpression '!'                                # forcedValueExpression
  | postfixExpression '?'                                         # optionalChainingExpression
  ;
 
 // GRAMMAR OF A FUNCTION CALL EXPRESSION
+functionCallArgumentClause: '(' functionCallArgumentList? ')' ;
+functionCallArgumentList: functionCallArgument (',' functionCallArgument)* ;
+// (expression | operator) is optional to handle selector expressions (see #425 for example)
+functionCallArgument: (functionCallIdentifier ':') (expression | operator)?
+  | (functionCallIdentifier ':')? (expression | operator) ;
+// SwiftLanguageReference does not have keyword
+functionCallIdentifier: identifier | keyword ;
 
-/*
-functionCallExpression
-  : postfixExpression parenthesizedExpression
-  : postfixExpression parenthesizedExpression? trailingClosure
-  ;
-  */
+// GRAMMAR OF AN ARGUMENT NAME
+argumentNames : argumentName+  ;
+argumentName: (identifier | '_') ':'  ; // Swift Language Reference has argumentName → identifier :
 
 //trailing_closure : closure_expression  ;
 
 //initializer_expression : postfix_expression '.' 'init' ;
 
-/*explicitMemberExpression
+/*
+explicitMemberExpression
   : postfixExpression '.' DecimalLiteral // TODO: don't allow '_' here in DecimalLiteral
   | postfixExpression '.' identifier genericArgumentClause?
+  | postfixExpression '.' identifier '(' argumentNames ')'
   ;
-  */
+*/
 
 //postfix_self_expression : postfix_expression '.' 'self' ;
 
@@ -703,7 +750,7 @@ functionCallExpression
 // are also referenced individually. For example, type signatures use
 // <...>.
 
-operatorHead: '=' | '<' | '>' | '!' | '*' | '&' | '==' | '?' | '-' | '&&' | '||' | '/' | OperatorHead;
+operatorHead: '=' | '<' | '>' | '!' | '*' | '&' | '==' | '?' | '-' | '&&' | '||' | '/' | '>=' | '->' | OperatorHead;
 operatorCharacter: operatorHead | OperatorCharacter;
 
 operator: operatorHead operatorCharacter*
@@ -758,16 +805,23 @@ postfixOperator : operator  ;
 sType
  : arrayType
  | dictionaryType
- | sType 'throws'? '->' sType  // function-type
- | sType 'rethrows' '->' sType // function-type
+ | functionType
  | typeIdentifier
  | tupleType
  | sType '?'  // optional-type
  | sType '!'  // implicitly-unwrapped-optional-type
  | protocolCompositionType
  | sType '.' 'Type' | sType '.' 'Protocol' // metatype
+ | 'Any' | 'Self'
  ;
 
+functionType: attributes? functionTypeArgumentClause ('throws' | 'rethrows')? '->' sType ;
+functionTypeArgumentClause: '(' ')'
+ | '(' functionTypeArgumentList '...'? ')' ;
+functionTypeArgumentList: functionTypeArgument (',' functionTypeArgument)* ;
+functionTypeArgument: attributes? 'inout'? sType | argumentLabel typeAnnotation ;
+argumentLabel: identifier ;
+
 arrayType: '[' sType ']' ;
 
 dictionaryType: '[' sType ':' sType ']' ;
@@ -778,31 +832,29 @@ implicitlyUnwrappedOptionalType: sType '!' ;
 
 // GRAMMAR OF A TYPE ANNOTATION
 
-typeAnnotation : ':' attributes? sType  ;
+typeAnnotation : ':' attributes? 'inout'? sType  ;
 
 // GRAMMAR OF A TYPE IDENTIFIER
 
 typeIdentifier
  : typeName genericArgumentClause?
  | typeName genericArgumentClause? '.' typeIdentifier
- | 'Self' // Swift Language Reference does not have this
  ;
 
 typeName : identifier ;
 
 // GRAMMAR OF A TUPLE TYPE
 
-tupleType : '('  tupleTypeBody? ')'  ;
-tupleTypeBody : tupleTypeElementList '...'? ;
-tupleTypeElementList : tupleTypeElement | tupleTypeElement ',' tupleTypeElementList  ;
-tupleTypeElement : attributes? 'inout'? sType | 'inout'? elementName typeAnnotation ;
+tupleType : '('  tupleTypeElementList? ')'  ;
+tupleTypeElementList : tupleTypeElement (',' tupleTypeElement)*  ;
+tupleTypeElement : elementName typeAnnotation | sType ;
 elementName : identifier  ;
 
 // GRAMMAR OF A PROTOCOL COMPOSITION TYPE
 
-protocolCompositionType : 'protocol' '<' protocolIdentifierList? '>'  ;
-protocolIdentifierList : protocolIdentifier | protocolIdentifier ',' protocolIdentifierList  ;
-protocolIdentifier : typeIdentifier  ;
+protocolCompositionType: protocolIdentifier '&' protocolCompositionContinuation ;
+protocolCompositionContinuation: protocolIdentifier | protocolCompositionType ;
+protocolIdentifier: typeIdentifier ;
 
 // GRAMMAR OF A METATYPE TYPE
 
@@ -817,25 +869,39 @@ typeInheritanceClause : ':' classRequirement ',' typeInheritanceList
 typeInheritanceList : typeIdentifier (',' typeIdentifier)* ;
 classRequirement: 'class' ;
 
-// ------ Build Configurations (Macros) -------
+// GRAMMAR OF A COMPILER CONTROL STATEMENT
 
-compilerControlStatement: buildConfigurationStatement | lineControlStatement ;
-buildConfigurationStatement: '#if' buildConfiguration statements? buildConfigurationElseIfClauses? buildConfigurationElseClause? '#endif' ;
-buildConfigurationElseIfClauses: buildConfigurationElseIfClause+ ;
-buildConfigurationElseIfClause: '#elseif' buildConfiguration statements? ;
-buildConfigurationElseClause: '#else' statements? ;
+compilerControlStatement: conditionalCompilationBlock | lineControlStatement ;
 
-buildConfiguration: platformTestingFunction | identifier | booleanLiteral
- | '(' buildConfiguration ')'
- | '!' buildConfiguration
- | buildConfiguration ('&&' | '||') buildConfiguration
+// GRAMMAR OF A CONDITIONAL COMPILATION BLOCK
+
+conditionalCompilationBlock: ifDirectiveClause elseifDirectiveClauses? elseDirectiveClause? '#endif' ;
+ifDirectiveClause: '#if' compilationCondition statements? ;
+elseifDirectiveClauses: elseifDirectiveClause+ ;
+elseifDirectiveClause: '#elseif' compilationCondition statements? ;
+elseDirectiveClause: '#else' statements? ;
+
+compilationCondition
+ : platformCondition
+ | identifier
+ | booleanLiteral
+ | '(' compilationCondition ')'
+ | '!' compilationCondition
+ | compilationCondition ('&&' | '||') compilationCondition
+ ;
+
+platformCondition
+ : 'os' '(' operatingSystem ')'
+ | 'arch' '(' architecture ')'
+ | 'swift' '(' '>=' swiftVersion ')'
  ;
 
-platformTestingFunction: 'os' '(' operatingSystem ')' | 'arch' '(' architecture ')' ;
 operatingSystem: 'OSX' | 'iOS' | 'watchOS' | 'tvOS' ;
 architecture: 'i386' | 'x86_64' | 'arm' | 'arm64' ;
+swiftVersion: FloatingPointLiteral ;
 
-lineControlStatement: '#line' (lineNumber fileName)? ;
+lineControlStatement: '#sourceLocation' '(' 'file' ':' fileName ',' 'line' ':' lineNumber ')'
+ | '#sourceLocation' '(' ')' ;
 lineNumber: integerLiteral ;
 fileName: StringLiteral ;
 
@@ -846,18 +912,38 @@ NilLiteral: 'nil' ;
 
 // GRAMMAR OF AN IDENTIFIER
 
-identifier : Identifier | contextSensitiveKeyword ;
+identifier : Identifier | contextSensitiveKeyword | grammarString ;
 
-keyword : 'convenience' | 'class' | 'deinit' | 'enum' | 'extension' | 'func' | 'import' | 'init' | 'let' | 'protocol' | 'static' | 'struct' | 'subscript' | 'typealias' | 'var' | 'break' | 'case' | 'continue' | 'default' | 'do' | 'else' | 'fallthrough' | 'if' | 'in' | 'for' | 'return' | 'switch' | 'where' | 'while' | 'as' | 'dynamicType' | 'is' | 'super' | 'self' | 'Self' | 'Type' | 'repeat' ;
+keyword :
+ // Keywords used in declarations
+ 'associatedtype' | 'class' | 'deinit' | 'enum' | 'extension' | 'fileprivate' | 'func' | 'import' | 'init' | 'inout'
+ | 'internal' | 'let' | 'open' | 'operator' | 'private' | 'protocol' | 'public' | 'static' | 'struct' | 'subscript'
+ | 'typealias' | 'var'
+ // Keywords used in statements
+ | 'break' | 'case' | 'continue' | 'default' | 'defer' | 'do' | 'else' | 'fallthrough' | 'for' | 'guard' | 'if' | 'in'
+ | 'repeat' | 'return' | 'switch' | 'where' | 'while'
+ // Keywords used in expressions and types
+ | 'as' | 'Any' | 'catch' | 'dynamicType' | 'is' | 'nil' | 'rethrows' | 'super' | 'self' | 'Self' | 'throw'
+ | 'throws' | 'try'
+ | BooleanLiteral
+ // Keywords used in patterns
+ | '_'
+ // Keywords that begin with a number sign (#)
+ | '#available' | '#colorLiteral' | '#column' | '#else' | '#elseif' | '#endif' | '#file' | 'fileLiteral' | '#function'
+ | '#if' | 'imageLiteral' | '#line' | '#selector'
+ ;
 
 contextSensitiveKeyword :
  'associativity' | 'convenience' | 'dynamic' | 'didSet' | 'final' | 'get' | 'infix' | 'indirect' |
  'lazy' | 'left' | 'mutating' | 'none' | 'nonmutating' | 'optional' | 'operator' | 'override' | 'postfix' | 'precedence' |
  'prefix' | 'Protocol' | 'required' | 'right' | 'set' | 'Type' | 'unowned' | 'weak' | 'willSet' |
- 'iOS' | 'iOSApplicationExtension' | 'OSX' | 'OSXApplicationExtension-' | 'watchOS' | 'x86_64' |
- 'arm' | 'arm64' | 'i386' | 'os' | 'arch'
+ 'iOS' | 'iOSApplicationExtension' | 'OSX' | 'OSXApplicationExtension­' | 'watchOS' | 'x86_64' |
+ 'arm' | 'arm64' | 'i386' | 'os' | 'arch' | 'safe' | 'tvOS' | 'file' | 'line' | 'default' | 'Self' | 'var'
  ;
 
+grammarString:
+  'red' | 'blue' | 'green' | 'alpha' | 'resourceName' | 'of' | 'type' ;
+
 OperatorHead
   : '/' | '=' | '-' | '+' | '!' | '*' | '%' | '<' | '>' | '&' | '|' | '^' | '~' | '?'
   | [\u00A1-\u00A7]
@@ -943,7 +1029,7 @@ integerLiteral
 BinaryLiteral : '0b' BinaryDigit BinaryLiteralCharacters? ;
 fragment BinaryDigit : [01] ;
 fragment BinaryLiteralCharacter : BinaryDigit | '_'  ;
-fragment BinaryLiteralCharacters : BinaryLiteralCharacter BinaryLiteralCharacters? ;
+fragment BinaryLiteralCharacters : BinaryLiteralCharacter+ ;
 
 OctalLiteral : '0o' OctalDigit OctalLiteralCharacters? ;
 fragment OctalDigit : [0-7] ;
@@ -979,12 +1065,14 @@ VersionLiteral: DecimalLiteral DecimalFraction DecimalFraction ;
 // GRAMMAR OF A STRING LITERAL
 
 StringLiteral : '"' QuotedText? '"' ;
-fragment QuotedText : QuotedTextItem QuotedText? ;
-fragment QuotedTextItem : EscapedCharacter
+fragment QuotedText : QuotedTextItem+ ;
+fragment QuotedTextItem : EscapedCharacter | InterpolatedString
 // | '\\(' expression ')'
  | ~["\\\u000A\u000D]
  ;
-EscapedCharacter : '\\' [0\\(tnr"']
+fragment InterpolatedString: '\\(' (QuotedTextItem | StringLiteral)* ')';
+
+EscapedCharacter : '\\' [0\\tnr"']
  | '\\x' HexadecimalDigit HexadecimalDigit
  | '\\u' '{' HexadecimalDigit HexadecimalDigit? HexadecimalDigit? HexadecimalDigit? HexadecimalDigit? HexadecimalDigit? HexadecimalDigit? HexadecimalDigit? '}'
 ;
diff --git a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
index ebad5ae5d7..8169e9fc0e 100644
--- a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
+++ b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
@@ -30,7 +30,7 @@ public class SwiftTokenizerTest extends AbstractTokenizerTest {
 
     @Test
     public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 3811;
+        this.expectedTokenCount = 3797;
         super.tokenizeTest();
     }
 }

From b10e21df7ff04b318a22baf5e6dac627f7454b81 Mon Sep 17 00:00:00 2001
From: kenji21 <r.bergoin@openium.fr>
Date: Fri, 8 Dec 2017 06:05:06 +0100
Subject: [PATCH 02/53] Add more swift4 syntax and update grammar for multiline

---
 .../pmd/lang/swift/antlr4/Swift.g4            | 21 ++++-
 .../pmd/cpd/SwiftTokenizerTest.java           |  2 +-
 .../net/sourceforge/pmd/cpd/BTree.swift       | 94 +++++++++++++++++++
 3 files changed, 111 insertions(+), 6 deletions(-)

diff --git a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4 b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
index 605800b350..d6277137dc 100644
--- a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
+++ b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
@@ -675,7 +675,7 @@ selectorExpression
 // GRAMMAR OF A KEY PATH EXPRESSION
 
 keyPathExpression
- : '#keyPath' '(' expression ')' 
+ : '#keyPath' '(' expression ')'
  | '\\' expression
 ;
 
@@ -903,7 +903,7 @@ swiftVersion: FloatingPointLiteral ;
 lineControlStatement: '#sourceLocation' '(' 'file' ':' fileName ',' 'line' ':' lineNumber ')'
  | '#sourceLocation' '(' ')' ;
 lineNumber: integerLiteral ;
-fileName: StringLiteral ;
+fileName: SingleStringLiteral ;
 
 // ---------- Lexical Structure -----------
 
@@ -1013,7 +1013,7 @@ ImplicitParameterName : '$' DecimalLiteral ; // TODO: don't allow '_' here
 // GRAMMAR OF A LITERAL
 
 booleanLiteral: BooleanLiteral ;
-literal : numericLiteral | StringLiteral | BooleanLiteral | NilLiteral ;
+literal : numericLiteral | MultiStringLiteral | SingleStringLiteral | BooleanLiteral | NilLiteral ;
 
 // GRAMMAR OF AN INTEGER LITERAL
 
@@ -1064,13 +1064,24 @@ VersionLiteral: DecimalLiteral DecimalFraction DecimalFraction ;
 
 // GRAMMAR OF A STRING LITERAL
 
-StringLiteral : '"' QuotedText? '"' ;
+TRIPLEDQUOTES : '"""' ;
+
+MultiStringLiteral : TRIPLEDQUOTES '\n' .*? '\n' TRIPLEDQUOTES ;
+fragment MultiQuotedText : MultiQuotedTextItem+ ;
+fragment MultiQuotedTextItem : MultiInterpolatedString
+ | ~[\\\u000A\u000D]
+ ;
+fragment MultiInterpolatedString: '\\(' (MultiQuotedTextItem | SingleStringLiteral)* ')';
+
+// StringLiteral : '"' QuotedText? '"' ;
+SingleStringLiteral : '"' QuotedText? '"' ;
+fragment SingleDoubleQuote : '"' | ~["] ;
 fragment QuotedText : QuotedTextItem+ ;
 fragment QuotedTextItem : EscapedCharacter | InterpolatedString
 // | '\\(' expression ')'
  | ~["\\\u000A\u000D]
  ;
-fragment InterpolatedString: '\\(' (QuotedTextItem | StringLiteral)* ')';
+fragment InterpolatedString: '\\(' (QuotedTextItem | SingleStringLiteral)* ')';
 
 EscapedCharacter : '\\' [0\\tnr"']
  | '\\x' HexadecimalDigit HexadecimalDigit
diff --git a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
index 8169e9fc0e..3a73f173f7 100644
--- a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
+++ b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
@@ -30,7 +30,7 @@ public class SwiftTokenizerTest extends AbstractTokenizerTest {
 
     @Test
     public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 3797;
+        this.expectedTokenCount = 4155;
         super.tokenizeTest();
     }
 }
diff --git a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
index 96731997ad..9c69edabaf 100644
--- a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
+++ b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
@@ -888,3 +888,97 @@ extension BTree {
         self.init(builder.finish())
     }
 }
+
+/// Swift 4 news
+
+
+extension BTree {
+
+    // Multi-line String Literals
+
+    func multiline() {
+        let star = "⭐️"
+        let introString = """
+          A long time ago in a galaxy far,
+          far away....
+
+          You could write multi-lined strings
+          without "escaping" single quotes.
+
+          The indentation of the closing quotes
+               below deside where the text line
+          begins.
+
+          You can even dynamically add values
+          from properties: \(star)
+          """
+        print(introString) // prints the string exactly as written above with the value of star
+    }
+
+    // One-Sided Ranges
+
+    func planets() {
+        var planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
+        let outsideAsteroidBelt = planets[4...] // Before: planets[4..<planets.endIndex]
+        let firstThree = planets[..<4]          // Before: planets[planets.startIndex..<4]
+
+        temperature(planetNumber: 3) // Earth
+    }
+
+    func temperature(planetNumber: Int) {
+      switch planetNumber {
+      case ...2: // anything less than or equal to 2
+        print("Too hot")
+      case 4...: // anything greater than or equal to 4
+        print("Too cold")
+      default:
+        print("Justtttt right")
+      }
+    }
+
+
+}
+
+// Generic Subscripts
+
+struct GenericDictionary<Key: Hashable, Value> {
+  private var data: [Key: Value]
+
+  init(data: [Key: Value]) {
+    self.data = data
+  }
+
+  subscript<T>(key: Key) -> T? {
+    return data[key] as? T
+  }
+}
+
+func useSomeGenericSubscript() {
+    // Dictionary of type: [String: Any]
+    var earthData = GenericDictionary(data: ["name": "Earth", "population": 7500000000, "moons": 1])
+
+    // Automatically infers return type without "as? String"
+    let name: String? = earthData["name"]
+
+    // Automatically infers return type without "as? Int"
+    let population: Int? = earthData["population"]
+}
+
+extension GenericDictionary {
+  subscript<Keys: Sequence>(keys: Keys) -> [Value] where Keys.Iterator.Element == Key {
+    var values: [Value] = []
+    for key in keys {
+      if let value = data[key] {
+        values.append(value)
+      }
+    }
+    return values
+  }
+}
+
+func useSomeGenericFromExtension() {
+    // Array subscript value
+    let nameAndMoons = earthData[["moons", "name"]]        // [1, "Earth"]
+    // Set subscript value
+    let nameAndMoons2 = earthData[Set(["moons", "name"])]  // [1, "Earth"]
+}

From 56cead842339dfa9a01c874cacb244b1991f492f Mon Sep 17 00:00:00 2001
From: kenji21 <r.bergoin@openium.fr>
Date: Fri, 8 Dec 2017 06:13:11 +0100
Subject: [PATCH 03/53] Add Associated Type Constraints & Class and Protocol
 Existential example

Update token count to match these additions already managed by the
grammar
---
 .../pmd/cpd/SwiftTokenizerTest.java           |  2 +-
 .../net/sourceforge/pmd/cpd/BTree.swift       | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
index 3a73f173f7..f09619dd01 100644
--- a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
+++ b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
@@ -30,7 +30,7 @@ public class SwiftTokenizerTest extends AbstractTokenizerTest {
 
     @Test
     public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 4155;
+        this.expectedTokenCount = 4217;
         super.tokenizeTest();
     }
 }
diff --git a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
index 9c69edabaf..33a60c91d8 100644
--- a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
+++ b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
@@ -982,3 +982,24 @@ func useSomeGenericFromExtension() {
     // Set subscript value
     let nameAndMoons2 = earthData[Set(["moons", "name"])]  // [1, "Earth"]
 }
+
+// Associated Type Constraints
+
+protocol MyProtocol {
+  associatedtype Element
+  associatedtype SubSequence : Sequence where SubSequence.Iterator.Element == Iterator.Element
+}
+
+// Class and Protocol Existential
+
+protocol MyProtocol { }
+class View { }
+class ViewSubclass: View, MyProtocol { }
+
+class MyClass {
+  var delegate: (View & MyProtocol)?
+}
+
+let myClass = MyClass()
+//myClass.delegate = View() // error: cannot assign value of type 'View' to type '(View & MyProtocol)?'
+myClass.delegate = ViewSubclass()

From 19f5110dc1c3479428b1c594e9be7d92cd433621 Mon Sep 17 00:00:00 2001
From: kenji21 <r.bergoin@openium.fr>
Date: Tue, 3 Apr 2018 10:22:42 +0200
Subject: [PATCH 04/53] Update Swift.g4 for swift 4.1

---
 .../net/sourceforge/pmd/lang/swift/antlr4/Swift.g4     |  3 +++
 .../net/sourceforge/pmd/cpd/SwiftTokenizerTest.java    |  2 +-
 .../test/resources/net/sourceforge/pmd/cpd/BTree.swift | 10 ++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4 b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
index d6277137dc..0eb9f7517e 100644
--- a/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
+++ b/pmd-swift/src/main/antlr4/net/sourceforge/pmd/lang/swift/antlr4/Swift.g4
@@ -894,11 +894,14 @@ platformCondition
  : 'os' '(' operatingSystem ')'
  | 'arch' '(' architecture ')'
  | 'swift' '(' '>=' swiftVersion ')'
+ | 'canImport' '(' moduleName ')'
+ | 'targetEnvironment' '(' 'simulator' ')'
  ;
 
 operatingSystem: 'OSX' | 'iOS' | 'watchOS' | 'tvOS' ;
 architecture: 'i386' | 'x86_64' | 'arm' | 'arm64' ;
 swiftVersion: FloatingPointLiteral ;
+moduleName: IdentifierCharacters ;
 
 lineControlStatement: '#sourceLocation' '(' 'file' ':' fileName ',' 'line' ':' lineNumber ')'
  | '#sourceLocation' '(' ')' ;
diff --git a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
index f09619dd01..5c6870686d 100644
--- a/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
+++ b/pmd-swift/src/test/java/net/sourceforge/pmd/cpd/SwiftTokenizerTest.java
@@ -30,7 +30,7 @@ public class SwiftTokenizerTest extends AbstractTokenizerTest {
 
     @Test
     public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 4217;
+        this.expectedTokenCount = 4239;
         super.tokenizeTest();
     }
 }
diff --git a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
index 33a60c91d8..9b283a3ec8 100644
--- a/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
+++ b/pmd-swift/src/test/resources/net/sourceforge/pmd/cpd/BTree.swift
@@ -1003,3 +1003,13 @@ class MyClass {
 let myClass = MyClass()
 //myClass.delegate = View() // error: cannot assign value of type 'View' to type '(View & MyProtocol)?'
 myClass.delegate = ViewSubclass()
+
+// 4.1 conditional
+
+#if canImport(SomeModule)
+let someModuleImportedVersion = SomeModule.version
+#endif
+
+#if targetEnvironment(simulator)
+print("code only compiled for simulator")
+#endif

From c77b0b44f3eb7628e508aada59b706b83855f1fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Tue, 10 Apr 2018 01:28:29 -0300
Subject: [PATCH 05/53] [apex] Stricter hardcoded ID detection

 - IDs are only 15, or 18 digits long
 - 18 digits long IDs are actually 15 digit IDs + checksum, which is now
validated
 - Resolves #776
---
 .../errorprone/AvoidHardcodingIdRule.java     | 49 ++++++++++++++++++-
 .../rule/errorprone/xml/AvoidHardcodingId.xml | 24 +++++++++
 2 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/errorprone/AvoidHardcodingIdRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/errorprone/AvoidHardcodingIdRule.java
index 80d3eee36d..248784713d 100644
--- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/errorprone/AvoidHardcodingIdRule.java
+++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/errorprone/AvoidHardcodingIdRule.java
@@ -4,18 +4,35 @@
 
 package net.sourceforge.pmd.lang.apex.rule.errorprone;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.regex.Pattern;
 
 import net.sourceforge.pmd.lang.apex.ast.ASTLiteralExpression;
 import net.sourceforge.pmd.lang.apex.rule.AbstractApexRule;
 
 public class AvoidHardcodingIdRule extends AbstractApexRule {
-    private static final Pattern PATTERN = Pattern.compile("^[a-zA-Z0-9]{5}[0][a-zA-Z0-9]{9,12}$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern PATTERN = Pattern.compile("^[a-zA-Z0-9]{5}0[a-zA-Z0-9]{9}([a-zA-Z0-5]{3})?$");
+    private static final Map<String, Character> CHECKSUM_LOOKUP;
+    
+    static {
+        final Map<String, Character> lookup = new HashMap<>();
+        final char[] chartable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345".toCharArray();
+        
+        for (int i = 0; i < chartable.length; i++) {
+            lookup.put(String.format("%5s", Integer.toBinaryString(i)).replace(' ', '0'), chartable[i]);
+        }
+        
+        CHECKSUM_LOOKUP = Collections.unmodifiableMap(lookup);
+    }
     
     public AvoidHardcodingIdRule() {
         setProperty(CODECLIMATE_CATEGORIES, "Style");
         setProperty(CODECLIMATE_REMEDIATION_MULTIPLIER, 100);
         setProperty(CODECLIMATE_BLOCK_HIGHLIGHTING, false);
+        
+        addRuleChainVisit(ASTLiteralExpression.class);
     }
 
     @Override
@@ -24,9 +41,39 @@ public class AvoidHardcodingIdRule extends AbstractApexRule {
         if (o instanceof String) {
             String literal = (String) o;
             if (PATTERN.matcher(literal).matches()) {
+                // 18-digit ids are just 15 digit ids + checksums, validate it  or it's not an id
+                if (literal.length() == 18 && !validateChecksum(literal)) {
+                    return data;
+                }
                 addViolation(data, node);
             }
         }
         return data;
     }
+
+    /*
+     * ID validation - sources:
+     * https://stackoverflow.com/questions/9742913/validating-a-salesforce-id#answer-29299786
+     * https://gist.github.com/jeriley/36b29f7c46527af4532aaf092c90dd56
+     */
+    private boolean validateChecksum(String literal) {
+        final String part1 = literal.substring(0, 5);
+        final String part2 = literal.substring(5, 10);
+        final String part3 = literal.substring(10, 15);
+        
+        final char checksum1 = checksum(part1);
+        final char checksum2 = checksum(part2);
+        final char checksum3 = checksum(part3);
+        
+        return literal.charAt(15) == checksum1 && literal.charAt(16) == checksum2
+                && literal.charAt(17) == checksum3;
+    }
+
+    private char checksum(String part) {
+        final StringBuilder sb = new StringBuilder(5);
+        for (int i = 4; i >= 0; i--) {
+            sb.append(Character.isUpperCase(part.charAt(i)) ? '1' : '0');
+        }
+        return CHECKSUM_LOOKUP.get(sb.toString());
+    }
 }
diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
index b7d77b9f05..8e8f7393de 100644
--- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
+++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
@@ -47,6 +47,30 @@ public class Foo {
 	void foo() {
 		return 'jatuatzbtazi124';
 	}
+}
+		]]></code>
+	</test-code>
+	
+	<test-code>
+		<description>Test for random string combinations - more than 15, less than 18 digits</description>
+		<expected-problems>0</expected-problems>
+		<code><![CDATA[
+public class Foo {
+	void foo() {
+		return 'jatua0tzbtazi1243';
+	}
+}
+		]]></code>
+	</test-code>
+	
+	<test-code>
+		<description>Test for random string combinations - checksum doesn't match</description>
+		<expected-problems>0</expected-problems>
+		<code><![CDATA[
+public class Foo {
+	void foo() {
+		return '001A0000006Vm9uIAE';
+	}
 }
 		]]></code>
 	</test-code>

From ffd88ef11ef33d85438f47b7d4eb42f6433a52a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Tue, 10 Apr 2018 01:31:51 -0300
Subject: [PATCH 06/53] Update changelog, refs #776

---
 docs/pages/release_notes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index ff4666b648..44716c719f 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -56,6 +56,8 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 
 *   all
     *   [#988](https://github.com/pmd/pmd/issues/988): \[core] FileNotFoundException for missing classes directory with analysis cache enabled
+*   apex-errorprone
+    *   [#776](https://github.com/pmd/pmd/issues/776): \[apex] AvoidHardcodingId false positives
 *   documentation
     *   [#994](https://github.com/pmd/pmd/issues/994): \[doc] Delete duplicate page contributing.md on the website
 *   java

From d6bea21ed6d5869efc843a948dd0a57e903ee30c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Wed, 11 Apr 2018 01:49:40 -0300
Subject: [PATCH 07/53] [core] Refactor CPD token filtering

 - Define a generic `TokenFilter` interface in pmd-core
 - Provide a base, extension-friendly `JavaCCTokenFilter` to process and
filter JavaCC token streams, honoring `CPD-OFF` and `CPD-ON` comments
 - Refactor the `JavaTokenizer` to use `JavaCCTokenFilter` by extending
it and adding custom Java-specific token filters
---
 .../pmd/cpd/token/JavaCCTokenFilter.java      | 85 ++++++++++++++++++
 .../pmd/cpd/token/TokenFilter.java            | 19 ++++
 .../sourceforge/pmd/lang/TokenManager.java    |  1 +
 .../sourceforge/pmd/cpd/JavaTokenizer.java    | 86 ++++++++-----------
 4 files changed, 141 insertions(+), 50 deletions(-)
 create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
 create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/TokenFilter.java

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
new file mode 100644
index 0000000000..0fb4a41a52
--- /dev/null
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
@@ -0,0 +1,85 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd.token;
+
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
+
+/**
+ * A generic filter for JavaCC-based token managers that allows to use comments
+ * to enable / disable analysis of parts of the stream
+ */
+public class JavaCCTokenFilter implements TokenFilter {
+
+    private final TokenManager tokenManager;
+    private boolean discardingSuppressing;
+    
+    /**
+     * Creates a new JavaCCTokenFilter
+     * @param tokenManager The token manager from which to retrieve tokens to be filtered
+     */
+    public JavaCCTokenFilter(final TokenManager tokenManager) {
+        this.tokenManager = tokenManager;
+    }
+
+    @Override
+    public final GenericToken getNextToken() {
+        GenericToken currentToken = (GenericToken) tokenManager.getNextToken();
+        while (!currentToken.getImage().isEmpty()) {
+            analyzeToken(currentToken);
+            processCPDSuppression(currentToken);
+
+            if (isDiscarding()) {
+                currentToken = (GenericToken) tokenManager.getNextToken();
+                continue;
+            }
+
+            return currentToken;
+        }
+        
+        return null;
+    }
+
+    private boolean isDiscarding() {
+        return discardingSuppressing || isLanguageSpecificDiscarding();
+    }
+
+    private void processCPDSuppression(final GenericToken currentToken) {
+        // Check if a comment is altering the suppression state
+        GenericToken comment = currentToken.getPreviousComment();
+        while (comment != null) {
+            if (comment.getImage().contains("CPD-OFF")) {
+                discardingSuppressing = true;
+                break;
+            }
+            if (comment.getImage().contains("CPD-ON")) {
+                discardingSuppressing = false;
+                break;
+            }
+            comment = comment.getPreviousComment();
+        }
+    }
+
+    /**
+     * Extension point for subclasses to indicate tokens are to be filtered.
+     * 
+     * @return True if tokens should be filtered, false otherwise
+     */
+    protected boolean isLanguageSpecificDiscarding() {
+        return false;
+    }
+    
+    /**
+     * Extension point for subclasses to analyze all tokens (before filtering)
+     * and update internal status to decide on custom discard rules.
+     * 
+     * @param currentToken The token to be analyzed
+     * @see #isLanguageSpecificDiscarding()
+     */
+    protected void analyzeToken(final GenericToken currentToken) {
+        // noop
+    }
+
+}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/TokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/TokenFilter.java
new file mode 100644
index 0000000000..965db0cbfd
--- /dev/null
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/TokenFilter.java
@@ -0,0 +1,19 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd.token;
+
+import net.sourceforge.pmd.lang.ast.GenericToken;
+
+/**
+ * Defines filter to be applied to the token stream during CPD analysis
+ */
+public interface TokenFilter {
+
+    /**
+     * Retrieves the next token to pass the filter
+     * @return The next token to pass the filter, or null if the end of the stream was reached
+     */
+    GenericToken getNextToken();
+}
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/TokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/TokenManager.java
index 6673d13a68..e0e67c78cc 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/TokenManager.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/TokenManager.java
@@ -8,6 +8,7 @@ package net.sourceforge.pmd.lang;
  * Common interface for interacting with parser Token Managers.
  */
 public interface TokenManager {
+    // TODO : Change the return to GenericToken in 7.0.0 - maybe even use generics TokenManager<T extends GenericToken>
     Object getNextToken();
 
     void setFileName(String fileName);
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
index 505c150996..4ed923a31a 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
@@ -9,9 +9,11 @@ import java.util.Deque;
 import java.util.LinkedList;
 import java.util.Properties;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
 import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
 import net.sourceforge.pmd.lang.java.JavaLanguageModule;
 import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;
 import net.sourceforge.pmd.lang.java.ast.Token;
@@ -31,34 +33,30 @@ public class JavaTokenizer implements Tokenizer {
         ignoreIdentifiers = Boolean.parseBoolean(properties.getProperty(IGNORE_IDENTIFIERS, "false"));
     }
 
+    @Override
     public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
-        StringBuilder stringBuilder = sourceCode.getCodeBuffer();
-
-        // Note that Java version is irrelevant for tokenizing
-        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME)
-                .getVersion("1.4").getLanguageVersionHandler();
-        String fileName = sourceCode.getFileName();
-        TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
-                .getTokenManager(fileName, new StringReader(stringBuilder.toString()));
-        Token currentToken = (Token) tokenMgr.getNextToken();
-
-        TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
-        ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
-
-        while (currentToken.image.length() > 0) {
-            discarder.updateState(currentToken);
-
-            if (discarder.isDiscarding()) {
-                currentToken = (Token) tokenMgr.getNextToken();
-                continue;
-            }
+        final String fileName = sourceCode.getFileName();
+        final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode);
+        final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
 
+        Token currentToken = (Token) tokenFilter.getNextToken();
+        while (currentToken != null) {
             processToken(tokenEntries, fileName, currentToken, constructorDetector);
-            currentToken = (Token) tokenMgr.getNextToken();
+            currentToken = (Token) tokenFilter.getNextToken();
         }
         tokenEntries.add(TokenEntry.getEOF());
     }
 
+    private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) {
+        final StringBuilder stringBuilder = sourceCode.getCodeBuffer();
+        // Note that Java version is irrelevant for tokenizing
+        final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME)
+                .getVersion("1.4").getLanguageVersionHandler();
+        final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
+                .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString()));
+        return new JavaTokenFilter(tokenMgr, ignoreAnnotations);
+    }
+
     private void processToken(Tokens tokenEntries, String fileName, Token currentToken,
             ConstructorDetector constructorDetector) {
         String image = currentToken.image;
@@ -93,15 +91,14 @@ public class JavaTokenizer implements Tokenizer {
     }
 
     /**
-     * The {@link TokenDiscarder} consumes token by token and maintains state.
-     * It can detect, whether the current token belongs to an annotation and
-     * whether the current token should be discarded by CPD.
+     * The {@link JavaTokenFilter} extends the {@link JavaCCTokenFilter} to discard
+     * Java-specific tokens.
      * <p>
      * By default, it discards semicolons, package and import statements, and
-     * enables CPD suppression. Optionally, all annotations can be ignored, too.
+     * enables annotation-based CPD suppression. Optionally, all annotations can be ignored, too.
      * </p>
      */
-    private static class TokenDiscarder {
+    private static class JavaTokenFilter extends JavaCCTokenFilter {
         private boolean isAnnotation = false;
         private boolean nextTokenEndsAnnotation = false;
         private int annotationStack = 0;
@@ -112,22 +109,24 @@ public class JavaTokenizer implements Tokenizer {
         private boolean discardingAnnotations = false;
         private boolean ignoreAnnotations = false;
 
-        TokenDiscarder(boolean ignoreAnnotations) {
+        JavaTokenFilter(final TokenManager tokenManager, final boolean ignoreAnnotations) {
+            super(tokenManager);
             this.ignoreAnnotations = ignoreAnnotations;
         }
 
-        public void updateState(Token currentToken) {
-            detectAnnotations(currentToken);
+        @Override
+        protected void analyzeToken(final GenericToken currentToken) {
+            detectAnnotations((Token) currentToken);
 
-            skipSemicolon(currentToken);
-            skipPackageAndImport(currentToken);
-            skipCPDSuppression(currentToken);
+            skipSemicolon((Token) currentToken);
+            skipPackageAndImport((Token) currentToken);
+            skipAnnotationSuppression((Token) currentToken);
             if (ignoreAnnotations) {
                 skipAnnotations();
             }
         }
 
-        private void skipPackageAndImport(Token currentToken) {
+        private void skipPackageAndImport(final Token currentToken) {
             if (currentToken.kind == JavaParserConstants.PACKAGE || currentToken.kind == JavaParserConstants.IMPORT) {
                 discardingKeywords = true;
             } else if (discardingKeywords && currentToken.kind == JavaParserConstants.SEMICOLON) {
@@ -135,7 +134,7 @@ public class JavaTokenizer implements Tokenizer {
             }
         }
 
-        private void skipSemicolon(Token currentToken) {
+        private void skipSemicolon(final Token currentToken) {
             if (currentToken.kind == JavaParserConstants.SEMICOLON) {
                 discardingSemicolon = true;
             } else if (discardingSemicolon && currentToken.kind != JavaParserConstants.SEMICOLON) {
@@ -143,21 +142,7 @@ public class JavaTokenizer implements Tokenizer {
             }
         }
 
-        private void skipCPDSuppression(Token currentToken) {
-            // Check if a comment is altering the suppression state
-            Token st = currentToken.specialToken;
-            while (st != null) {
-                if (st.image.contains("CPD-OFF")) {
-                    discardingSuppressing = true;
-                    break;
-                }
-                if (st.image.contains("CPD-ON")) {
-                    discardingSuppressing = false;
-                    break;
-                }
-                st = st.specialToken;
-            }
-
+        private void skipAnnotationSuppression(final Token currentToken) {
             // if processing an annotation, look for a CPD-START or CPD-END
             if (isAnnotation) {
                 if (!discardingSuppressing && currentToken.kind == JavaParserConstants.STRING_LITERAL
@@ -178,7 +163,8 @@ public class JavaTokenizer implements Tokenizer {
             }
         }
 
-        public boolean isDiscarding() {
+        @Override
+        protected boolean isLanguageSpecificDiscarding() {
             return discardingSemicolon || discardingKeywords || discardingAnnotations
                     || discardingSuppressing;
         }

From 47e345945ed1b589a6b19f6ae6e5b9d573f7bfd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sat, 14 Apr 2018 22:33:37 -0300
Subject: [PATCH 08/53] [cpp] Allow CPD suppressions with comments

---
 pmd-cpp/etc/grammar/cpp.jj                    | 33 +++++++------------
 .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 20 ++++++-----
 .../sourceforge/pmd/cpd/CPPTokenizerTest.java | 15 +++++++--
 3 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/pmd-cpp/etc/grammar/cpp.jj b/pmd-cpp/etc/grammar/cpp.jj
index 3f65d72a8e..a39effe982 100644
--- a/pmd-cpp/etc/grammar/cpp.jj
+++ b/pmd-cpp/etc/grammar/cpp.jj
@@ -129,10 +129,9 @@ public final class CppParser {
       return sym.IsCtor(GetFullyScopedName());
   }
 }
-
 PARSER_END(CppParser)
 
-SKIP :
+SKIP:
 {
   " "
 |
@@ -143,38 +142,28 @@ SKIP :
   "\r\n"
 |
   "\n"
-|
-  "//" : IN_LINE_COMMENT
-|
-  "/*"  : IN_COMMENT
 |
   "#" : PREPROCESSOR_OUTPUT
 }
 
-<IN_LINE_COMMENT> SKIP:
-{
-   "\n" : DEFAULT
-}
+<DEFAULT,PREPROCESSOR_OUTPUT> SPECIAL_TOKEN:
+{ <SINGLE_LINE_COMMENT: "//"(~["\n","\r"])* ("\n"|"\r"|"\r\n")?> }
 
-<IN_LINE_COMMENT> MORE:
-{
-  < ~[] >
-}
+MORE:
+{ "/*" : IN_MULTI_LINE_COMMENT }
 
-<IN_COMMENT> SKIP:
-{ "*/" : DEFAULT }
+<IN_MULTI_LINE_COMMENT> SPECIAL_TOKEN:
+{ <MULTI_LINE_COMMENT: "*/">: DEFAULT }
 
-<IN_COMMENT,IN_PREPROCESSOR_OUTPUT_COMMENT> MORE:
+<IN_MULTI_LINE_COMMENT,IN_PREPROCESSOR_OUTPUT_COMMENT> MORE:
 { < ~[] > }
 
-<IN_PREPROCESSOR_OUTPUT_COMMENT> SKIP:
-{ "*/" : PREPROCESSOR_OUTPUT }
+<IN_PREPROCESSOR_OUTPUT_COMMENT> SPECIAL_TOKEN:
+{ <PREPROCESSOR_OUTPUT_COMMENT: "*/">: PREPROCESSOR_OUTPUT }
 
 <PREPROCESSOR_OUTPUT> SKIP:
 {
    "\n" : DEFAULT
-   | "/*" : IN_PREPROCESSOR_OUTPUT_COMMENT
-   | "//" : IN_LINE_COMMENT
 }
 
 <PREPROCESSOR_OUTPUT> MORE:
@@ -183,6 +172,8 @@ SKIP :
   |
   "\\\r\n"
   |
+  "/*": IN_PREPROCESSOR_OUTPUT_COMMENT 
+  |
   < ~[] >
 }
 
diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
index 3a9b7b6fd2..2f49579c06 100644
--- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
+++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
@@ -13,12 +13,13 @@ import java.util.Properties;
 import org.apache.commons.io.IOUtils;
 
 import net.sourceforge.pmd.PMD;
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.GenericToken;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
-import net.sourceforge.pmd.lang.cpp.ast.Token;
 import net.sourceforge.pmd.util.IOUtil;
 
 /**
@@ -61,13 +62,14 @@ public class CPPTokenizer implements Tokenizer {
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(maybeSkipBlocks(buffer.toString()));
             reader = IOUtil.skipBOM(reader);
-            TokenManager tokenManager = languageVersionHandler
-                    .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
-                tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+            final TokenFilter tokenFilter = new JavaCCTokenFilter(
+                languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            
+            GenericToken currentToken = tokenFilter.getNextToken();
+            while (currentToken != null) {
+                tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine()));
+                currentToken = tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());
diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
index cc185a627f..23e3246cea 100644
--- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
+++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java
@@ -5,6 +5,7 @@
 package net.sourceforge.pmd.cpd;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Properties;
@@ -19,7 +20,7 @@ public class CPPTokenizerTest {
     @Test
     public void testUTFwithBOM() {
         Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
-        assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
         assertEquals(15, tokens.size());
     }
 
@@ -29,9 +30,19 @@ public class CPPTokenizerTest {
                 + "int main()\n" + "{\n" + "    std::string text(\"ąęćśźńó\");\n" + "    std::cout << text;\n"
                 + "    return 0;\n" + "}\n";
         Tokens tokens = parse(code);
-        assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
         assertEquals(24, tokens.size());
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() {
+        String code = "#include <iostream>\n" + "#include <string>\n" + "\n" + "// CPD-OFF\n"
+                + "int main()\n" + "{\n" + "    std::string text(\"ąęćśźńó\");\n" + "    std::cout << text;\n"
+                + "    return 0;\n" + "// CPD-ON\n" + "}\n";
+        Tokens tokens = parse(code);
+        assertNotSame(TokenEntry.getEOF(), tokens.getTokens().get(0));
+        assertEquals(2, tokens.size()); // "}" + EOF
+    }
 
     @Test
     public void testMultiLineMacros() {

From e838179bb2c20efb19acbeef668d192b66bd6bdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 01:44:02 -0300
Subject: [PATCH 09/53] Fix PMD dogfooding

---
 .../net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java   | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
index 0fb4a41a52..0b1cd53b4e 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/token/JavaCCTokenFilter.java
@@ -31,12 +31,11 @@ public class JavaCCTokenFilter implements TokenFilter {
             analyzeToken(currentToken);
             processCPDSuppression(currentToken);
 
-            if (isDiscarding()) {
-                currentToken = (GenericToken) tokenManager.getNextToken();
-                continue;
+            if (!isDiscarding()) {
+                return currentToken;
             }
 
-            return currentToken;
+            currentToken = (GenericToken) tokenManager.getNextToken();
         }
         
         return null;

From 78f7343713dd1fc5f954c08b5591053d41489026 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 02:21:28 -0300
Subject: [PATCH 10/53] [javascript] Support comment based CPD suppressions

---
 .../sourceforge/pmd/cpd/EcmascriptTokenizer.java | 13 +++++++------
 .../pmd/cpd/EcmascriptTokenizerTest.java         | 16 ++++++++++++++++
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
index 2d27aa5897..9fcecbc903 100644
--- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
+++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java
@@ -9,9 +9,10 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.ecmascript.EcmascriptLanguageModule;
 import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5ParserConstants;
@@ -30,14 +31,14 @@ public class EcmascriptTokenizer implements Tokenizer {
             LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME)
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(buffer.toString());
-            TokenManager tokenManager = languageVersionHandler
+            TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
                     .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            Token currentToken = (Token) tokenFilter.getNextToken();
+            while (currentToken != null) {
                 tokenEntries.add(
                         new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+                currentToken = (Token) tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());
diff --git a/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java b/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java
index 5f57357387..edf4dafdf6 100644
--- a/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java
+++ b/pmd-javascript/src/test/java/net/sourceforge/pmd/cpd/EcmascriptTokenizerTest.java
@@ -33,6 +33,22 @@ public class EcmascriptTokenizerTest {
         t.tokenize(sourceCode, tokens);
         assertEquals(45, tokens.size());
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        final String code = "// CPD-OFF" + PMD.EOL
+            + "function switchToRealPassword() {" + PMD.EOL
+            + "var real = $('realPass');" + PMD.EOL
+            + " var prompt = $('promptPass');" + PMD.EOL
+            + "// CPD-ON" + PMD.EOL
+            + "}" + PMD.EOL;
+        
+        Tokenizer t = new EcmascriptTokenizer();
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(code));
+        Tokens tokens = new Tokens();
+        t.tokenize(sourceCode, tokens);
+        assertEquals(2, tokens.size()); // Only "}" and EOL
+    }
 
     /**
      * See: https://sourceforge.net/p/pmd/bugs/1239/

From cf9b768b90e11e022b2cbd4baad6419702abf532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 02:58:04 -0300
Subject: [PATCH 11/53] [matlab] Support comment based CPD suppressions

---
 pmd-matlab/etc/grammar/matlab.jj              | 26 +++++++------------
 .../sourceforge/pmd/cpd/MatlabTokenizer.java  | 13 +++++-----
 .../pmd/cpd/MatlabTokenizerTest.java          | 20 ++++++++++++++
 3 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/pmd-matlab/etc/grammar/matlab.jj b/pmd-matlab/etc/grammar/matlab.jj
index 35dbc8625d..6f61dd0d42 100644
--- a/pmd-matlab/etc/grammar/matlab.jj
+++ b/pmd-matlab/etc/grammar/matlab.jj
@@ -39,27 +39,19 @@ PARSER_END(MatlabParser)
   "\r\n" : DEFAULT
 |
   "\n" : DEFAULT
-|
-  "%{"  : IN_COMMENT
-|
-  "%" : IN_LINE_COMMENT
-
 }
 
-<IN_COMMENT> SKIP:
-{
-    "%}" : DEFAULT
-}
+MORE:
+{ "%{": IN_COMMENT }
 
-<IN_LINE_COMMENT> SKIP:
-{
-   "\n" : DEFAULT
-}
+SPECIAL_TOKEN:
+{ <SINGLE_LINE_COMMENT: "%"(~["\n","\r"])* ("\n"|"\r"|"\r\n")?> }
 
-<IN_LINE_COMMENT, IN_COMMENT> MORE:
-{
-    < ~[] >
-}
+<IN_COMMENT> SPECIAL_TOKEN:
+{ <MULTI_LINE_COMMENT: "%}">: DEFAULT }
+
+<IN_COMMENT> MORE:
+{ < ~[] > }
 
 <DEFAULT, TRANSPOSE> TOKEN : /* SEPARATORS AND OTHER USEFULL LANGUAGE CONSTRUCTS*/
 {
diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
index b2157bd043..4b135d1be7 100644
--- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
+++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
@@ -9,9 +9,10 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.matlab.MatlabLanguageModule;
 import net.sourceforge.pmd.lang.matlab.ast.Token;
@@ -31,13 +32,13 @@ public class MatlabTokenizer implements Tokenizer {
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(buffer.toString());
             reader = IOUtil.skipBOM(reader);
-            TokenManager tokenManager = languageVersionHandler
+            final TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
                     .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            Token currentToken = (Token) tokenFilter.getNextToken();
+            while (currentToken != null) {
                 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+                currentToken = (Token) tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());
diff --git a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java b/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java
index e2ce74f064..0557063eb0 100644
--- a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java
+++ b/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java
@@ -4,12 +4,15 @@
 
 package net.sourceforge.pmd.cpd;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
 
 public class MatlabTokenizerTest extends AbstractTokenizerTest {
@@ -33,4 +36,21 @@ public class MatlabTokenizerTest extends AbstractTokenizerTest {
         this.expectedTokenCount = 3925;
         super.tokenizeTest();
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("% CPD-OFF" + PMD.EOL
+                + "function g = vec(op, y)" + PMD.EOL
+                + "  opy = op(y);" + PMD.EOL
+                + "  if ( any(size(opy) > 1) )" + PMD.EOL
+                + "    g = @loopWrapperArray;" + PMD.EOL
+                + "  end" + PMD.EOL
+                + "  % CPD-ON" + PMD.EOL
+                + "end"
+        ));
+        Tokens tokens = new Tokens();
+        tokenizer.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        assertEquals(2, tokens.size()); // 2 tokens: "end" + EOF
+    }
 }

From 94f00f4baf07136cf128e58d0e77709fd0d3bcaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 03:07:48 -0300
Subject: [PATCH 12/53] [obj-c] Add support for CPD comment-based suppressions

---
 .../pmd/cpd/ObjectiveCTokenizer.java            | 13 +++++++------
 .../pmd/cpd/UnicodeObjectiveCTokenizerTest.java | 17 +++++++++++++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
index 27dab7e2e5..b976c90008 100644
--- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
+++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java
@@ -9,9 +9,10 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.objectivec.ObjectiveCLanguageModule;
 import net.sourceforge.pmd.lang.objectivec.ast.Token;
@@ -29,13 +30,13 @@ public class ObjectiveCTokenizer implements Tokenizer {
             LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME)
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(buffer.toString());
-            TokenManager tokenManager = languageVersionHandler
+            final TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
                     .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            Token currentToken = (Token) tokenFilter.getNextToken();
+            while (currentToken != null) {
                 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+                currentToken = (Token) tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode.getFileName());
diff --git a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
index 7418a99b49..99ff788ba9 100644
--- a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
+++ b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
@@ -4,12 +4,15 @@
 
 package net.sourceforge.pmd.cpd;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
 
 //Tests if the ObjectiveC tokenizer supports identifiers with unicode characters
@@ -34,4 +37,18 @@ public class UnicodeObjectiveCTokenizerTest extends AbstractTokenizerTest {
         this.expectedTokenCount = 10;
         super.tokenizeTest();
     }
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
+            "// CPD-OFF" + PMD.EOL
+            + "static SecCertificateRef gNСServerLogonCertificate;" +  PMD.EOL
+            + "// CPD-ON" + PMD.EOL
+            + "@end" + PMD.EOL
+        ));
+        Tokens tokens = new Tokens();
+        tokenizer.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        assertEquals(2, tokens.size()); // 2 tokens: "@end" + EOF
+    }
 }

From b965090cc7a4c1253611bd25d717f211a61ed291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 14:21:42 -0300
Subject: [PATCH 13/53] [plsql] Support comment-based CPD suppressions

---
 .../sourceforge/pmd/cpd/PLSQLTokenizer.java   | 30 +++++++------------
 .../pmd/cpd/PLSQLTokenizerTest.java           | 21 +++++++++++--
 2 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
index 711b4305dd..74cfb32a41 100644
--- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
+++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java
@@ -9,19 +9,21 @@ import java.util.Properties;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
+import net.sourceforge.pmd.lang.plsql.PLSQLTokenManager;
 import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserConstants;
-import net.sourceforge.pmd.lang.plsql.ast.PLSQLParserTokenManager;
 import net.sourceforge.pmd.lang.plsql.ast.Token;
 
 public class PLSQLTokenizer implements Tokenizer {
     private static final Logger LOGGER = Logger.getLogger(PLSQLTokenizer.class.getName());
 
+    // This is actually useless, the comments are special tokens, never taken into account by CPD
+    @Deprecated
     public static final String IGNORE_COMMENTS = "ignore_comments";
     public static final String IGNORE_IDENTIFIERS = "ignore_identifiers";
     public static final String IGNORE_LITERALS = "ignore_literals";
 
-    private boolean ignoreComments;
     private boolean ignoreIdentifiers;
     private boolean ignoreLiterals;
 
@@ -32,13 +34,13 @@ public class PLSQLTokenizer implements Tokenizer {
          * interested in comment variation, so we shall default ignoreComments
          * to true
          */
-        ignoreComments = Boolean.parseBoolean(properties.getProperty(IGNORE_COMMENTS, "true"));
         ignoreIdentifiers = Boolean.parseBoolean(properties.getProperty(IGNORE_IDENTIFIERS, "false"));
         ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
     }
 
+    @Deprecated
     public void setIgnoreComments(boolean ignore) {
-        this.ignoreComments = ignore;
+        // This is actually useless, the comments are special tokens, never taken into account by CPD
     }
 
     public void setIgnoreLiterals(boolean ignore) {
@@ -65,7 +67,6 @@ public class PLSQLTokenizer implements Tokenizer {
         long addedTokens = 0;
 
         if (LOGGER.isLoggable(Level.FINE)) {
-            LOGGER.fine("PLSQLTokenizer: ignoreComments==" + ignoreComments);
             LOGGER.fine("PLSQLTokenizer: ignoreIdentifiers==" + ignoreIdentifiers);
             LOGGER.fine("PLSQLTokenizer: ignoreLiterals==" + ignoreLiterals);
         }
@@ -73,21 +74,12 @@ public class PLSQLTokenizer implements Tokenizer {
         String fileName = sourceCode.getFileName();
         StringBuilder sb = sourceCode.getCodeBuffer();
 
-        PLSQLParserTokenManager tokenMgr = new PLSQLParserTokenManager(
-                new SimpleCharStream(new StringReader(sb.toString())));
-        Token currentToken = tokenMgr.getNextToken();
-        while (currentToken.image.length() > 0) {
+        TokenFilter tokenFilter = new JavaCCTokenFilter(new PLSQLTokenManager(new StringReader(sb.toString())));
+        Token currentToken = (Token) tokenFilter.getNextToken();
+        while (currentToken != null) {
             String image = currentToken.image;
 
             encounteredTokens++;
-            if (ignoreComments && (currentToken.kind == PLSQLParserConstants.SINGLE_LINE_COMMENT
-                    || currentToken.kind == PLSQLParserConstants.MULTI_LINE_COMMENT
-                    || currentToken.kind == PLSQLParserConstants.FORMAL_COMMENT
-                    || currentToken.kind == PLSQLParserConstants.COMMENT
-                    || currentToken.kind == PLSQLParserConstants.IN_MULTI_LINE_COMMENT
-                    || currentToken.kind == PLSQLParserConstants.IN_FORMAL_COMMENT)) {
-                image = String.valueOf(currentToken.kind);
-            }
 
             if (ignoreIdentifiers && currentToken.kind == PLSQLParserConstants.IDENTIFIER) {
                 image = String.valueOf(currentToken.kind);
@@ -104,7 +96,7 @@ public class PLSQLTokenizer implements Tokenizer {
 
             tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
             addedTokens++;
-            currentToken = tokenMgr.getNextToken();
+            currentToken = (Token) tokenFilter.getNextToken();
         }
         tokenEntries.add(TokenEntry.getEOF());
         if (LOGGER.isLoggable(Level.FINE)) {
diff --git a/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java b/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java
index ace4b0517c..96d36dbd91 100644
--- a/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java
+++ b/pmd-plsql/src/test/java/net/sourceforge/pmd/cpd/PLSQLTokenizerTest.java
@@ -4,12 +4,15 @@
 
 package net.sourceforge.pmd.cpd;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
 
 public class PLSQLTokenizerTest extends AbstractTokenizerTest {
@@ -33,8 +36,20 @@ public class PLSQLTokenizerTest extends AbstractTokenizerTest {
         this.expectedTokenCount = 1422;
         super.tokenizeTest();
     }
-
-    public static junit.framework.Test suite() {
-        return new junit.framework.JUnit4TestAdapter(PLSQLTokenizerTest.class);
+    
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("-- CPD-OFF" + PMD.EOL
+                + "CREATE OR REPLACE" + PMD.EOL
+                + "PACKAGE \"test_schema\".\"BANK_DATA\"" + PMD.EOL
+                + "IS" + PMD.EOL
+                + "pi      CONSTANT NUMBER := 3.1415;" + PMD.EOL
+                + "--CPD-ON" + PMD.EOL
+                + "END;"
+        ));
+        Tokens tokens = new Tokens();
+        tokenizer.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        assertEquals(3, tokens.size()); // 3 tokens: "END" + ";" + EOF
     }
 }

From 539773eb5bd07d9ad28c203852080e8388f7908f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 15 Apr 2018 15:37:40 -0300
Subject: [PATCH 14/53] Fix PMD dogfooding

---
 .../net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
index 99ff788ba9..2a0d510fcc 100644
--- a/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
+++ b/pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java
@@ -42,7 +42,7 @@ public class UnicodeObjectiveCTokenizerTest extends AbstractTokenizerTest {
     public void testIgnoreBetweenSpecialComments() throws IOException {
         SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
             "// CPD-OFF" + PMD.EOL
-            + "static SecCertificateRef gNСServerLogonCertificate;" +  PMD.EOL
+            + "static SecCertificateRef gNСServerLogonCertificate;" + PMD.EOL
             + "// CPD-ON" + PMD.EOL
             + "@end" + PMD.EOL
         ));

From 7c7a66d7503ea0c28fcb0b4b2dc060741c35e3ef Mon Sep 17 00:00:00 2001
From: Andreas Bergander <andreas.bergander@benify.com>
Date: Tue, 17 Apr 2018 17:27:52 +0200
Subject: [PATCH 15/53] Make BasicProjectMemoizer thread safe. Fixes issue
 #1020.

---
 .../pmd/lang/metrics/BasicProjectMemoizer.java     | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/metrics/BasicProjectMemoizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/metrics/BasicProjectMemoizer.java
index efc8e06c74..e5b6a7463d 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/metrics/BasicProjectMemoizer.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/metrics/BasicProjectMemoizer.java
@@ -28,6 +28,8 @@ public abstract class BasicProjectMemoizer<T extends QualifiableNode, O extends
     private Map<QualifiedName, MetricMemoizer<T>> classes = new WeakHashMap<>();
     private Map<QualifiedName, MetricMemoizer<O>> operations = new WeakHashMap<>();
 
+    private final Object classesSynchronizer = new Object();
+    private final Object operationsSynchronizer = new Object();
 
     /** Clears all memoizers. Used for tests. */
     public void reset() {
@@ -38,8 +40,10 @@ public abstract class BasicProjectMemoizer<T extends QualifiableNode, O extends
 
     @Override
     public MetricMemoizer<O> getOperationMemoizer(QualifiedName qname) {
-        if (!operations.containsKey(qname)) {
-            operations.put(qname, new BasicMetricMemoizer<O>());
+        synchronized (operationsSynchronizer) {
+            if (!operations.containsKey(qname)) {
+                operations.put(qname, new BasicMetricMemoizer<O>());
+            }
         }
 
         return operations.get(qname);
@@ -48,8 +52,10 @@ public abstract class BasicProjectMemoizer<T extends QualifiableNode, O extends
 
     @Override
     public MetricMemoizer<T> getClassMemoizer(QualifiedName qname) {
-        if (!classes.containsKey(qname)) {
-            classes.put(qname, new BasicMetricMemoizer<T>());
+        synchronized (classesSynchronizer) {
+            if (!classes.containsKey(qname)) {
+                classes.put(qname, new BasicMetricMemoizer<T>());
+            }
         }
 
         return classes.get(qname);

From a3f30476cef48bed34a47a4bbd878090db74f533 Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 15:02:05 -0700
Subject: [PATCH 16/53] Hard-coded IV used in crypto operations

---
 .../rule/security/InsecureCryptoIvRule.java   | 136 ++++++++++++++++++
 .../category/java/categories.properties       |   6 +-
 .../main/resources/category/java/security.xml |  23 +++
 .../java/rule/security/SecurityRulesTest.java |  14 ++
 .../rule/security/xml/InsecureCryptoIv.xml    |  80 +++++++++++
 5 files changed, 255 insertions(+), 4 deletions(-)
 create mode 100644 pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
 create mode 100644 pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
 create mode 100644 pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
new file mode 100644
index 0000000000..e058ecfb71
--- /dev/null
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -0,0 +1,136 @@
+package net.sourceforge.pmd.lang.java.rule.security;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import net.sourceforge.pmd.lang.java.ast.ASTArrayInitializer;
+import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceDeclaration;
+import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType;
+import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit;
+import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration;
+import net.sourceforge.pmd.lang.java.ast.ASTLiteral;
+import net.sourceforge.pmd.lang.java.ast.ASTLocalVariableDeclaration;
+import net.sourceforge.pmd.lang.java.ast.ASTName;
+import net.sourceforge.pmd.lang.java.ast.ASTPrimitiveType;
+import net.sourceforge.pmd.lang.java.ast.ASTReferenceType;
+import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer;
+import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
+
+/**
+ * Finds hardcoded static Initialization Vectors vectors used with cryptographic
+ * operations.
+ * 
+ * //bad: byte[] ivBytes = new byte[] {32, 87, -14, 25, 78, -104, 98, 40};
+ * //bad: byte[] ivBytes = "hardcoded".getBytes(); //bad: byte[] ivBytes =
+ * someString.getBytes();
+ * 
+ * javax.crypto.spec.IvParameterSpec must not be created from a static
+ * 
+ * @author sergeygorbaty
+ *
+ */
+public class InsecureCryptoIvRule extends AbstractJavaRule {
+
+	public InsecureCryptoIvRule() {
+
+		addRuleChainVisit(ASTCompilationUnit.class);
+		addRuleChainVisit(ASTLiteral.class);
+	}
+
+	@Override
+	public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
+		Set<ASTFieldDeclaration> foundFields = new HashSet<>();
+		Set<ASTLocalVariableDeclaration> foundLocalVars = new HashSet<>();
+		Set<String> passedInIvVarNames = new HashSet<>();
+
+		// byte[] fields
+		List<ASTFieldDeclaration> fields = node.findDescendantsOfType(ASTFieldDeclaration.class);
+		for (ASTFieldDeclaration field : fields) {
+			foundFields.addAll(extractPrimitiveTypes(field));
+		}
+
+		List<ASTLocalVariableDeclaration> localVars = node.findDescendantsOfType(ASTLocalVariableDeclaration.class);
+		for (ASTLocalVariableDeclaration localVar : localVars) {
+			// byte[] local vars
+			foundLocalVars.addAll(extractPrimitiveTypes(localVar));
+
+			// find javax.crypto.spec.IvParameterSpec
+			ASTClassOrInterfaceType declClassName = localVar.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
+			if (declClassName != null) {
+				Class<?> foundClass = declClassName.getTypeDefinition() == null ? null
+						: declClassName.getTypeDefinition().getType();
+
+				if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
+					ASTVariableInitializer init = localVar.getFirstDescendantOfType(ASTVariableInitializer.class);
+					if (init != null) {
+						ASTName name = init.getFirstDescendantOfType(ASTName.class);
+						if (name != null) {
+							passedInIvVarNames.add(name.getImage());
+						}
+					}
+
+				}
+			}
+
+			for (ASTFieldDeclaration foundField : foundFields) {
+				if (passedInIvVarNames.contains(foundField.getVariableName())) {
+					validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
+				}
+			}
+
+			for (ASTLocalVariableDeclaration foundLocalVar : foundLocalVars) {
+				if (passedInIvVarNames.contains(foundLocalVar.getVariableName())) {
+					validateProperIv(data, foundLocalVar.getFirstDescendantOfType(ASTVariableInitializer.class));
+				}
+			}
+
+		}
+
+		return data;
+	}
+
+	private Set<ASTLocalVariableDeclaration> extractPrimitiveTypes(ASTLocalVariableDeclaration localVar) {
+		List<ASTPrimitiveType> types = localVar.findDescendantsOfType(ASTPrimitiveType.class);
+		Set<ASTLocalVariableDeclaration> retVal = new HashSet<>();
+		extractPrimitiveTypesInner(retVal, localVar, types);
+
+		return retVal;
+	}
+
+	private Set<ASTFieldDeclaration> extractPrimitiveTypes(ASTFieldDeclaration field) {
+		List<ASTPrimitiveType> types = field.findDescendantsOfType(ASTPrimitiveType.class);
+		Set<ASTFieldDeclaration> retVal = new HashSet<>();
+
+		extractPrimitiveTypesInner(retVal, field, types);
+
+		return retVal;
+	}
+
+	private <T> void extractPrimitiveTypesInner(Set<T> retVal, T field, List<ASTPrimitiveType> types) {
+		for (ASTPrimitiveType type : types) {
+			if (type.hasImageEqualTo("byte")) {
+				ASTReferenceType parent = type.getFirstParentOfType(ASTReferenceType.class);
+				if (parent != null) {
+					retVal.add(field);
+				}
+			}
+		}
+	}
+
+	private void validateProperIv(Object data, ASTVariableInitializer varInit) {
+		// hard coded array
+		ASTArrayInitializer arrayInit = varInit.getFirstDescendantOfType(ASTArrayInitializer.class);
+		if (arrayInit != null) {
+			addViolation(data, varInit);
+		}
+
+		// string literal
+		ASTLiteral literal = varInit.getFirstDescendantOfType(ASTLiteral.class);
+		if (literal != null && literal.isStringLiteral()) {
+			addViolation(data, varInit);
+		}
+
+	}
+
+}
diff --git a/pmd-java/src/main/resources/category/java/categories.properties b/pmd-java/src/main/resources/category/java/categories.properties
index cc92c1fe94..3189fd3ade 100644
--- a/pmd-java/src/main/resources/category/java/categories.properties
+++ b/pmd-java/src/main/resources/category/java/categories.properties
@@ -9,7 +9,5 @@ rulesets.filenames=\
     category/java/documentation.xml,\
     category/java/errorprone.xml,\
     category/java/multithreading.xml,\
-    category/java/performance.xml
-
-#   security doesn't contain any rules yet
-#    category/java/security.xml
+    category/java/performance.xml,\
+    category/java/security.xml
diff --git a/pmd-java/src/main/resources/category/java/security.xml b/pmd-java/src/main/resources/category/java/security.xml
index 92c805257d..15eed88266 100644
--- a/pmd-java/src/main/resources/category/java/security.xml
+++ b/pmd-java/src/main/resources/category/java/security.xml
@@ -8,5 +8,28 @@
     <description>
 Rules that flag potential security flaws.
     </description>
+    
+        <rule name="InsecureCryptoIv"
+          since="6.1"
+          message="Do not use hard coded initialization vector in crypto operations"
+          class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
+          externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
+        <description>
+Do not use hard coded initialization vector in cryptographic operations. Please use randomly generated IV.
+        </description>
+        <priority>3</priority>
+        <example>
+<![CDATA[
+public class Foo {
+    void good() {
+    }
+
+    void bad() {
+    }
+}
+]]>
+        </example>
+    </rule>
+    
 
 </ruleset>
\ No newline at end of file
diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
new file mode 100644
index 0000000000..59023d1e6b
--- /dev/null
+++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
@@ -0,0 +1,14 @@
+package net.sourceforge.pmd.lang.java.rule.security;
+
+import net.sourceforge.pmd.testframework.SimpleAggregatorTst;
+
+public class SecurityRulesTest extends SimpleAggregatorTst {
+
+    private static final String RULESET = "category/java/security.xml";
+
+    @Override
+    public void setUp() {
+        addRule(RULESET, "InsecureCryptoIv");
+    }
+
+}
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
new file mode 100644
index 0000000000..c3379c6422
--- /dev/null
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<test-data xmlns="http://pmd.sourceforge.net/rule-tests"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://pmd.sourceforge.net/rule-tests http://pmd.sourceforge.net/rule-tests_1_0_0.xsd">
+	<test-code>
+		<description><![CDATA[
+Hard coded Iv from string, bad
+     ]]></description>
+		<expected-problems>1</expected-problems>
+		<code><![CDATA[
+ import javax.crypto.spec.IvParameterSpec;       
+ 
+ public class Foo {
+
+ 	void encrypt() {
+		byte[] staticIv = "ALL_ZEROS_HERE".getBytes();
+		IvParameterSpec iv = new IvParameterSpec(staticIv);
+	}
+}
+     ]]></code>
+	</test-code>
+
+	<test-code>
+		<description><![CDATA[
+Hard coded Iv field, bad
+     ]]></description>
+		<expected-problems>1</expected-problems>
+		<code><![CDATA[
+ import javax.crypto.spec.IvParameterSpec;       
+ 
+ public class Foo {
+ 	
+ 	byte[] ivBytes = new byte[] { 32, 87, -14, 25, 78, -104, 98, 40 };
+ 	
+ 	void encrypt() {
+		
+		IvParameterSpec iv = new IvParameterSpec(ivBytes);
+	}
+}
+     ]]></code>
+	</test-code>
+	<test-code>
+		<description><![CDATA[
+Hard coded Iv local var, bad
+     ]]></description>
+		<expected-problems>1</expected-problems>
+		<code><![CDATA[
+ import javax.crypto.spec.IvParameterSpec;       
+ 
+ public class Foo {
+
+ 	void encrypt() {
+		byte[] ivBytes = new byte[] { 32, 87, -14, 25, 78, -104, 98, 40 };
+		IvParameterSpec iv = new IvParameterSpec(ivBytes);
+	}
+}
+     ]]></code>
+	</test-code>
+
+
+	<test-code>
+		<description><![CDATA[
+Random IV, good
+     ]]></description>
+		<expected-problems>0</expected-problems>
+		<code><![CDATA[
+import javax.crypto.spec.IvParameterSpec;        
+import java.security.SecureRandom;        
+        
+public class Foo {
+ 	void encrypt() {
+		byte[] iv = new byte[16];
+		SecureRandom sprng = new SecureRandom();
+		sprng.nextBytes(iv); 
+		IvParameterSpec ivs = new IvParameterSpec(ivBytes);
+	}	
+}
+     ]]></code>
+	</test-code>
+</test-data>

From 1f7fc0ac8b05e786c004ad43545980f8cfcd9d39 Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 15:27:58 -0700
Subject: [PATCH 17/53] Indentation and license

---
 .../rule/security/InsecureCryptoIvRule.java   | 167 +++++++++---------
 .../java/rule/security/SecurityRulesTest.java |   4 +
 2 files changed, 89 insertions(+), 82 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
index e058ecfb71..91e3dfb913 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -1,3 +1,7 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
 package net.sourceforge.pmd.lang.java.rule.security;
 
 import java.util.HashSet;
@@ -25,112 +29,111 @@ import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
  * //bad: byte[] ivBytes = "hardcoded".getBytes(); //bad: byte[] ivBytes =
  * someString.getBytes();
  * 
- * javax.crypto.spec.IvParameterSpec must not be created from a static
+ * javax.crypto.spec.IvParameterSpec must not be created from a static sources
  * 
  * @author sergeygorbaty
+ * @since 6.3
  *
  */
 public class InsecureCryptoIvRule extends AbstractJavaRule {
 
-	public InsecureCryptoIvRule() {
+    public InsecureCryptoIvRule() {
+        addRuleChainVisit(ASTCompilationUnit.class);
+    }
 
-		addRuleChainVisit(ASTCompilationUnit.class);
-		addRuleChainVisit(ASTLiteral.class);
-	}
+    @Override
+    public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
+        Set<ASTFieldDeclaration> foundFields = new HashSet<>();
+        Set<ASTLocalVariableDeclaration> foundLocalVars = new HashSet<>();
+        Set<String> passedInIvVarNames = new HashSet<>();
 
-	@Override
-	public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
-		Set<ASTFieldDeclaration> foundFields = new HashSet<>();
-		Set<ASTLocalVariableDeclaration> foundLocalVars = new HashSet<>();
-		Set<String> passedInIvVarNames = new HashSet<>();
+        // byte[] fields
+        List<ASTFieldDeclaration> fields = node.findDescendantsOfType(ASTFieldDeclaration.class);
+        for (ASTFieldDeclaration field : fields) {
+            foundFields.addAll(extractPrimitiveTypes(field));
+        }
 
-		// byte[] fields
-		List<ASTFieldDeclaration> fields = node.findDescendantsOfType(ASTFieldDeclaration.class);
-		for (ASTFieldDeclaration field : fields) {
-			foundFields.addAll(extractPrimitiveTypes(field));
-		}
+        List<ASTLocalVariableDeclaration> localVars = node.findDescendantsOfType(ASTLocalVariableDeclaration.class);
+        for (ASTLocalVariableDeclaration localVar : localVars) {
+            // byte[] local vars
+            foundLocalVars.addAll(extractPrimitiveTypes(localVar));
 
-		List<ASTLocalVariableDeclaration> localVars = node.findDescendantsOfType(ASTLocalVariableDeclaration.class);
-		for (ASTLocalVariableDeclaration localVar : localVars) {
-			// byte[] local vars
-			foundLocalVars.addAll(extractPrimitiveTypes(localVar));
+            // find javax.crypto.spec.IvParameterSpec
+            ASTClassOrInterfaceType declClassName = localVar.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
+            if (declClassName != null) {
+                Class<?> foundClass = declClassName.getTypeDefinition() == null ? null
+                        : declClassName.getTypeDefinition().getType();
 
-			// find javax.crypto.spec.IvParameterSpec
-			ASTClassOrInterfaceType declClassName = localVar.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
-			if (declClassName != null) {
-				Class<?> foundClass = declClassName.getTypeDefinition() == null ? null
-						: declClassName.getTypeDefinition().getType();
+                if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
+                    ASTVariableInitializer init = localVar.getFirstDescendantOfType(ASTVariableInitializer.class);
+                    if (init != null) {
+                        ASTName name = init.getFirstDescendantOfType(ASTName.class);
+                        if (name != null) {
+                            passedInIvVarNames.add(name.getImage());
+                        }
+                    }
 
-				if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
-					ASTVariableInitializer init = localVar.getFirstDescendantOfType(ASTVariableInitializer.class);
-					if (init != null) {
-						ASTName name = init.getFirstDescendantOfType(ASTName.class);
-						if (name != null) {
-							passedInIvVarNames.add(name.getImage());
-						}
-					}
+                }
+            }
 
-				}
-			}
+            for (ASTFieldDeclaration foundField : foundFields) {
+                if (passedInIvVarNames.contains(foundField.getVariableName())) {
+                    validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
+                }
+            }
 
-			for (ASTFieldDeclaration foundField : foundFields) {
-				if (passedInIvVarNames.contains(foundField.getVariableName())) {
-					validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
-				}
-			}
+            for (ASTLocalVariableDeclaration foundLocalVar : foundLocalVars) {
+                if (passedInIvVarNames.contains(foundLocalVar.getVariableName())) {
+                    validateProperIv(data, foundLocalVar.getFirstDescendantOfType(ASTVariableInitializer.class));
+                }
+            }
 
-			for (ASTLocalVariableDeclaration foundLocalVar : foundLocalVars) {
-				if (passedInIvVarNames.contains(foundLocalVar.getVariableName())) {
-					validateProperIv(data, foundLocalVar.getFirstDescendantOfType(ASTVariableInitializer.class));
-				}
-			}
+        }
 
-		}
+        return data;
+    }
 
-		return data;
-	}
+    private Set<ASTLocalVariableDeclaration> extractPrimitiveTypes(ASTLocalVariableDeclaration localVar) {
+        List<ASTPrimitiveType> types = localVar.findDescendantsOfType(ASTPrimitiveType.class);
+        Set<ASTLocalVariableDeclaration> retVal = new HashSet<>();
+        extractPrimitiveTypesInner(retVal, localVar, types);
 
-	private Set<ASTLocalVariableDeclaration> extractPrimitiveTypes(ASTLocalVariableDeclaration localVar) {
-		List<ASTPrimitiveType> types = localVar.findDescendantsOfType(ASTPrimitiveType.class);
-		Set<ASTLocalVariableDeclaration> retVal = new HashSet<>();
-		extractPrimitiveTypesInner(retVal, localVar, types);
+        return retVal;
+    }
 
-		return retVal;
-	}
+    private Set<ASTFieldDeclaration> extractPrimitiveTypes(ASTFieldDeclaration field) {
+        List<ASTPrimitiveType> types = field.findDescendantsOfType(ASTPrimitiveType.class);
+        Set<ASTFieldDeclaration> retVal = new HashSet<>();
 
-	private Set<ASTFieldDeclaration> extractPrimitiveTypes(ASTFieldDeclaration field) {
-		List<ASTPrimitiveType> types = field.findDescendantsOfType(ASTPrimitiveType.class);
-		Set<ASTFieldDeclaration> retVal = new HashSet<>();
+        extractPrimitiveTypesInner(retVal, field, types);
 
-		extractPrimitiveTypesInner(retVal, field, types);
+        return retVal;
+    }
 
-		return retVal;
-	}
+    private <T> void extractPrimitiveTypesInner(Set<T> retVal, T field, List<ASTPrimitiveType> types) {
+        for (ASTPrimitiveType type : types) {
+            if (type.hasImageEqualTo("byte")) {
+                ASTReferenceType parent = type.getFirstParentOfType(ASTReferenceType.class);
+                if (parent != null) {
+                    retVal.add(field);
+                }
+            }
+        }
+    }
 
-	private <T> void extractPrimitiveTypesInner(Set<T> retVal, T field, List<ASTPrimitiveType> types) {
-		for (ASTPrimitiveType type : types) {
-			if (type.hasImageEqualTo("byte")) {
-				ASTReferenceType parent = type.getFirstParentOfType(ASTReferenceType.class);
-				if (parent != null) {
-					retVal.add(field);
-				}
-			}
-		}
-	}
+    private void validateProperIv(Object data, ASTVariableInitializer varInit) {
+        // hard coded array
+        ASTArrayInitializer arrayInit = varInit.getFirstDescendantOfType(ASTArrayInitializer.class);
+        if (arrayInit != null) {
+            addViolation(data, varInit);
+        }
 
-	private void validateProperIv(Object data, ASTVariableInitializer varInit) {
-		// hard coded array
-		ASTArrayInitializer arrayInit = varInit.getFirstDescendantOfType(ASTArrayInitializer.class);
-		if (arrayInit != null) {
-			addViolation(data, varInit);
-		}
+        // string literal
+        ASTLiteral literal = varInit.getFirstDescendantOfType(ASTLiteral.class);
+        if (literal != null && literal.isStringLiteral()) {
+            addViolation(data, varInit);
+        }
 
-		// string literal
-		ASTLiteral literal = varInit.getFirstDescendantOfType(ASTLiteral.class);
-		if (literal != null && literal.isStringLiteral()) {
-			addViolation(data, varInit);
-		}
-
-	}
+    }
 
 }
diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
index 59023d1e6b..2a71bd3481 100644
--- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
+++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/security/SecurityRulesTest.java
@@ -1,3 +1,7 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
 package net.sourceforge.pmd.lang.java.rule.security;
 
 import net.sourceforge.pmd.testframework.SimpleAggregatorTst;

From 475744b515601f8621edfb02f99e6e46d70a4a39 Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 15:32:33 -0700
Subject: [PATCH 18/53] Code example

---
 .../src/main/resources/category/java/security.xml   | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pmd-java/src/main/resources/category/java/security.xml b/pmd-java/src/main/resources/category/java/security.xml
index 15eed88266..67b89c28e5 100644
--- a/pmd-java/src/main/resources/category/java/security.xml
+++ b/pmd-java/src/main/resources/category/java/security.xml
@@ -10,22 +10,31 @@ Rules that flag potential security flaws.
     </description>
     
         <rule name="InsecureCryptoIv"
-          since="6.1"
+          since="6.3"
           message="Do not use hard coded initialization vector in crypto operations"
           class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
           externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
         <description>
-Do not use hard coded initialization vector in cryptographic operations. Please use randomly generated IV.
+Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV.
         </description>
         <priority>3</priority>
         <example>
 <![CDATA[
 public class Foo {
     void good() {
+        SecureRandom random = new SecureRandom();
+        byte iv[] = new byte[16];
+        random.nextBytes(bytes);
     }
 
     void bad() {
+        byte[] iv = new byte[] { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, };
     }
+    
+    void alsoBad() {
+        byte[] iv = "secret iv in here".getBytes();
+    }
+    
 }
 ]]>
         </example>

From 04bf04af104325dce11adbe59fe123e99c11176f Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 15:36:33 -0700
Subject: [PATCH 19/53] Simple example

---
 .../pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
index c3379c6422..0a9ff9b7b4 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
@@ -30,7 +30,7 @@ Hard coded Iv field, bad
  
  public class Foo {
  	
- 	byte[] ivBytes = new byte[] { 32, 87, -14, 25, 78, -104, 98, 40 };
+ 	byte[] ivBytes = new byte[] { { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, }; 
  	
  	void encrypt() {
 		

From 9d360ced035b557dd72f6dcb419f8cdeb053873a Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 15:37:00 -0700
Subject: [PATCH 20/53] all zeros

---
 .../pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
index 0a9ff9b7b4..9a4b16610c 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
@@ -30,7 +30,7 @@ Hard coded Iv field, bad
  
  public class Foo {
  	
- 	byte[] ivBytes = new byte[] { { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, }; 
+ 	byte[] ivBytes = new byte[] { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, }; 
  	
  	void encrypt() {
 		
@@ -50,7 +50,7 @@ Hard coded Iv local var, bad
  public class Foo {
 
  	void encrypt() {
-		byte[] ivBytes = new byte[] { 32, 87, -14, 25, 78, -104, 98, 40 };
+		byte[] ivBytes = new byte[] { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, }; 
 		IvParameterSpec iv = new IvParameterSpec(ivBytes);
 	}
 }

From 86e95d876c72b4123881be1b0cab2e8717564c4b Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 16:47:13 -0700
Subject: [PATCH 21/53] Refactoring

---
 .../rule/security/InsecureCryptoIvRule.java   | 38 ++++++-----
 .../rule/security/xml/InsecureCryptoIv.xml    | 63 ++++++++++++++++++-
 2 files changed, 82 insertions(+), 19 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
index 91e3dfb913..5e676393c7 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -8,6 +8,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import net.sourceforge.pmd.lang.java.ast.ASTAllocationExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTArrayInitializer;
 import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType;
@@ -16,6 +17,7 @@ import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTLiteral;
 import net.sourceforge.pmd.lang.java.ast.ASTLocalVariableDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTName;
+import net.sourceforge.pmd.lang.java.ast.ASTPrimaryExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTPrimitiveType;
 import net.sourceforge.pmd.lang.java.ast.ASTReferenceType;
 import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer;
@@ -53,19 +55,19 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
             foundFields.addAll(extractPrimitiveTypes(field));
         }
 
-        List<ASTLocalVariableDeclaration> localVars = node.findDescendantsOfType(ASTLocalVariableDeclaration.class);
-        for (ASTLocalVariableDeclaration localVar : localVars) {
-            // byte[] local vars
-            foundLocalVars.addAll(extractPrimitiveTypes(localVar));
+        // find new javax.crypto.spec.IvParameterSpec(...)
+        List<ASTAllocationExpression> allocations = node.findDescendantsOfType(ASTAllocationExpression.class);
+        for (ASTAllocationExpression allocation : allocations) {
 
-            // find javax.crypto.spec.IvParameterSpec
-            ASTClassOrInterfaceType declClassName = localVar.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
+            ASTClassOrInterfaceType declClassName = allocation.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
             if (declClassName != null) {
                 Class<?> foundClass = declClassName.getTypeDefinition() == null ? null
                         : declClassName.getTypeDefinition().getType();
 
                 if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
-                    ASTVariableInitializer init = localVar.getFirstDescendantOfType(ASTVariableInitializer.class);
+                    // ASTVariableInitializer init =
+                    // allocation.getFirstDescendantOfType(ASTVariableInitializer.class);
+                    ASTPrimaryExpression init = allocation.getFirstDescendantOfType(ASTPrimaryExpression.class);
                     if (init != null) {
                         ASTName name = init.getFirstDescendantOfType(ASTName.class);
                         if (name != null) {
@@ -75,19 +77,23 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
 
                 }
             }
+        }
 
-            for (ASTFieldDeclaration foundField : foundFields) {
-                if (passedInIvVarNames.contains(foundField.getVariableName())) {
-                    validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
-                }
+        List<ASTLocalVariableDeclaration> localVars = node.findDescendantsOfType(ASTLocalVariableDeclaration.class);
+        for (ASTLocalVariableDeclaration localVar : localVars) {
+            foundLocalVars.addAll(extractPrimitiveTypes(localVar));
+        }
+
+        for (ASTFieldDeclaration foundField : foundFields) {
+            if (passedInIvVarNames.contains(foundField.getVariableName())) {
+                validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
             }
+        }
 
-            for (ASTLocalVariableDeclaration foundLocalVar : foundLocalVars) {
-                if (passedInIvVarNames.contains(foundLocalVar.getVariableName())) {
-                    validateProperIv(data, foundLocalVar.getFirstDescendantOfType(ASTVariableInitializer.class));
-                }
+        for (ASTLocalVariableDeclaration foundLocalVar : foundLocalVars) {
+            if (passedInIvVarNames.contains(foundLocalVar.getVariableName())) {
+                validateProperIv(data, foundLocalVar.getFirstDescendantOfType(ASTVariableInitializer.class));
             }
-
         }
 
         return data;
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
index 9a4b16610c..5cfa996a0b 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
@@ -4,6 +4,43 @@
 	xsi:schemaLocation="http://pmd.sourceforge.net/rule-tests http://pmd.sourceforge.net/rule-tests_1_0_0.xsd">
 	<test-code>
 		<description><![CDATA[
+Hard coded inline IvSpec, bad
+     ]]></description>
+		<expected-problems>1</expected-problems>
+		<code><![CDATA[
+ import javax.crypto.spec.IvParameterSpec;       
+ import javax.crypto.Cipher;   
+ 
+ public class Foo {
+
+ 	void encrypt() {
+		byte[] ivBytes = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; 
+		Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
+		cipher.init(Cipher.ENCRYPT_MODE, keySpec, new IvParameterSpec(ivBytes));		 
+	}
+}
+     ]]></code>
+	</test-code>
+    <test-code>
+		<description><![CDATA[
+Randomly generated IV, good 
+     ]]></description>
+		<expected-problems>0</expected-problems>
+		<code><![CDATA[
+ import javax.crypto.spec.IvParameterSpec;       
+ 
+ public class Foo {
+
+ 	void encrypt(SecretKeySpec key) {
+ 	    byte[] ivBytes = new byte[key.getEncoded().length];
+        Util.getSecureRandom().nextBytes(ivBytes);
+		IvParameterSpec iv = new IvParameterSpec(ivBytes);
+	}
+}
+     ]]></code>
+	</test-code>
+	<test-code>
+		<description><![CDATA[
 Hard coded Iv from string, bad
      ]]></description>
 		<expected-problems>1</expected-problems>
@@ -19,7 +56,6 @@ Hard coded Iv from string, bad
 }
      ]]></code>
 	</test-code>
-
 	<test-code>
 		<description><![CDATA[
 Hard coded Iv field, bad
@@ -56,8 +92,6 @@ Hard coded Iv local var, bad
 }
      ]]></code>
 	</test-code>
-
-
 	<test-code>
 		<description><![CDATA[
 Random IV, good
@@ -77,4 +111,27 @@ public class Foo {
 }
      ]]></code>
 	</test-code>
+	<test-code>
+		<description><![CDATA[
+Inline IvSpec Random IV, good
+     ]]></description>
+		<expected-problems>0</expected-problems>
+		<code><![CDATA[
+import javax.crypto.spec.IvParameterSpec;        
+import java.security.SecureRandom;     
+import javax.crypto.Cipher;   
+        
+public class Foo {
+ 	void encrypt() {
+		byte[] iv = new byte[16];
+		SecureRandom sprng = new SecureRandom();
+		sprng.nextBytes(iv);
+		Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
+		cipher.init(Cipher.ENCRYPT_MODE, keySpec, new IvParameterSpec(iv));		 
+	}	
+}
+     ]]></code>
+	</test-code>
+
+
 </test-data>

From d3afc40c6798f71ef4d6c8f7a931d542b5de629b Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Tue, 17 Apr 2018 17:00:02 -0700
Subject: [PATCH 22/53] Remove unused code

---
 .../pmd/lang/java/rule/security/InsecureCryptoIvRule.java      | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
index 5e676393c7..3266c7661c 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -65,8 +65,6 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
                         : declClassName.getTypeDefinition().getType();
 
                 if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
-                    // ASTVariableInitializer init =
-                    // allocation.getFirstDescendantOfType(ASTVariableInitializer.class);
                     ASTPrimaryExpression init = allocation.getFirstDescendantOfType(ASTPrimaryExpression.class);
                     if (init != null) {
                         ASTName name = init.getFirstDescendantOfType(ASTName.class);
@@ -74,7 +72,6 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
                             passedInIvVarNames.add(name.getImage());
                         }
                     }
-
                 }
             }
         }

From 73a18d5fbd19ebe8fc8ee8b92f26120fc2ec4261 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Wed, 18 Apr 2018 09:26:26 +0200
Subject: [PATCH 23/53] Update surefire plugin to support building with java10

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d3e46304bc..195b8863aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -261,7 +261,7 @@ Additionally it includes CPD, the copy-paste-detector. CPD finds duplicated code
         <maven.compiler.target>1.${java.version}</maven.compiler.target>
 
         <javacc.version>5.0</javacc.version>
-        <surefire.version>2.20.1</surefire.version>
+        <surefire.version>2.21.0</surefire.version>
         <checkstyle.version>3.0.0</checkstyle.version>
         <pmd.plugin.version>3.9.0</pmd.plugin.version>
         <ant.version>1.10.1</ant.version>

From 6c6f64467573628eea15c92d541b888cf582e9d5 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Wed, 18 Apr 2018 09:48:46 +0200
Subject: [PATCH 24/53] [ci] Switch to OpenJDK 10

See https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html
---
 .travis.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7e1e0b8fd9..c5ceed7685 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,16 +1,11 @@
 dist: trusty
 sudo: false
 addons:
-  apt:
-    packages:
-      - oracle-java9-installer
   ssh_known_hosts:
     - web.sourceforge.net
 
 language: java
 
-jdk: oraclejdk9
-
 env:
   global:
   - secure: KBEuB6U1p5RQXSYe157AwydFr/zpXQPA0IChVCgZV+X1mMyy9ZtrjH1J1AXuviseDDXDbaT25sRnsvpl82rfRw2xOkMGXHy4N95/ylTSr8DjHxTao71BhXsvFycNobFva5y2EGNWqDvpS8I2oSZo7Qk4la3yep3rcJQvcy6RDbbhpDTbL1QMFyadunIBm0WtqbunrMqtjSqaoPsXz8TiQuxHvX4vEXzVbaxV1QQt79Vi+daa6wAV3mRQAugnx+UffsC8JqMxgm06usWeJgCJzxgm8E7clZCLmf53B2TL8dK6bIYbqyvOY3uFxitsTG0d8Z0GOJwXBgZNgbniTRO8ZJSty5eZP8LBybbjVLSL25DNTWtCjADUL/uySnXIEidlMt2N/3QmH7zrGAfAk/tIwKpdRca2GLLydeXf6PSkiahnPEkIY/QupcsOLELhdifpdOjb8QW1OenA+vUbNM9dccLwKnX6Fj9cu4VQG601AcYDr2eyhq8WYkr3wYdw/6KdUa3hmplowTBs+qguppP+eOSgGuEsy38KLtqnvm6WlHy6tcLmcVYKG3DmR1b7TWXsOXC6/VMH8BHBkvsF1QdRg9+Cgx07vX3Hw7roPiYzmaO9Ajs20ATsUfRskMuWCTeTSK5pN8X27veRCZlhFjeKQMDdmfVwzpAfRgKsl3TEn1I=
@@ -31,9 +26,11 @@ matrix:
   fast_finish: true
 
 before_install:
+  - wget https://github.com/sormuras/bach/raw/master/install-jdk.sh
   - bash .travis/setup-secrets.sh
   - bash .travis/configure-maven.sh
-install: true
+# Install OpenJDK 10 - see https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html
+install: . ./install-jdk.sh -F 10 -L GPL
 before_script: true
 script: source .travis/build-$BUILD.sh
 after_success: true

From 22a50a14cfb13fecc6c9efa06c7932e97b0f9506 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Wed, 18 Apr 2018 10:05:53 +0200
Subject: [PATCH 25/53] [ci] switch to Oracle jdk10 to support javafx

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c5ceed7685..7cd272880c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,8 +29,8 @@ before_install:
   - wget https://github.com/sormuras/bach/raw/master/install-jdk.sh
   - bash .travis/setup-secrets.sh
   - bash .travis/configure-maven.sh
-# Install OpenJDK 10 - see https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html
-install: . ./install-jdk.sh -F 10 -L GPL
+# Install OracleJDK 10 - see https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html
+install: . ./install-jdk.sh -F 10 -L BCL
 before_script: true
 script: source .travis/build-$BUILD.sh
 after_success: true

From 3d9c63a271f0d3400fccba25980b1f1f3548f243 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Wed, 18 Apr 2018 10:28:56 +0200
Subject: [PATCH 26/53] Update jacoco plugin for java10 compatibility

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 195b8863aa..58da6009b4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -476,7 +476,7 @@ Additionally it includes CPD, the copy-paste-detector. CPD finds duplicated code
                 <plugin>
                       <groupId>org.jacoco</groupId>
                       <artifactId>jacoco-maven-plugin</artifactId>
-                      <version>0.7.9</version>
+                      <version>0.8.1</version>
                 </plugin>
                 <!--This plugin's configuration is used to store Eclipse
                     m2e settings only. It has no influence on the Maven build itself. -->

From f9ebe4c11986b1034faedb80cd57e07434b3ecc5 Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Wed, 18 Apr 2018 10:25:12 -0700
Subject: [PATCH 27/53] Version change and use of shorthands

---
 .../rule/security/InsecureCryptoIvRule.java   | 11 +++----
 .../main/resources/category/java/security.xml | 29 +++++++++----------
 2 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
index 3266c7661c..95e9593b84 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -12,7 +12,6 @@ import net.sourceforge.pmd.lang.java.ast.ASTAllocationExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTArrayInitializer;
 import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType;
-import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit;
 import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTLiteral;
 import net.sourceforge.pmd.lang.java.ast.ASTLocalVariableDeclaration;
@@ -34,13 +33,13 @@ import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
  * javax.crypto.spec.IvParameterSpec must not be created from a static sources
  * 
  * @author sergeygorbaty
- * @since 6.3
+ * @since 6.3.0
  *
  */
 public class InsecureCryptoIvRule extends AbstractJavaRule {
 
     public InsecureCryptoIvRule() {
-        addRuleChainVisit(ASTCompilationUnit.class);
+        addRuleChainVisit(ASTClassOrInterfaceDeclaration.class);
     }
 
     @Override
@@ -61,10 +60,8 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
 
             ASTClassOrInterfaceType declClassName = allocation.getFirstDescendantOfType(ASTClassOrInterfaceType.class);
             if (declClassName != null) {
-                Class<?> foundClass = declClassName.getTypeDefinition() == null ? null
-                        : declClassName.getTypeDefinition().getType();
-
-                if (foundClass != null && foundClass.equals(javax.crypto.spec.IvParameterSpec.class)) {
+                Class<?> foundClass = declClassName.getType();
+                if (foundClass != null && javax.crypto.spec.IvParameterSpec.class.isAssignableFrom(foundClass)) {
                     ASTPrimaryExpression init = allocation.getFirstDescendantOfType(ASTPrimaryExpression.class);
                     if (init != null) {
                         ASTName name = init.getFirstDescendantOfType(ASTName.class);
diff --git a/pmd-java/src/main/resources/category/java/security.xml b/pmd-java/src/main/resources/category/java/security.xml
index 67b89c28e5..d148f3cdee 100644
--- a/pmd-java/src/main/resources/category/java/security.xml
+++ b/pmd-java/src/main/resources/category/java/security.xml
@@ -1,24 +1,23 @@
 <?xml version="1.0"?>
 
 <ruleset name="Security"
-    xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+	xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
 
-    <description>
+	<description>
 Rules that flag potential security flaws.
     </description>
-    
-        <rule name="InsecureCryptoIv"
-          since="6.3"
-          message="Do not use hard coded initialization vector in crypto operations"
-          class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
-          externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
-        <description>
+
+	<rule name="InsecureCryptoIv" since="6.3.0"
+		message="Do not use hard coded initialization vector in crypto operations"
+		class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
+		externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
+		<description>
 Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV.
         </description>
-        <priority>3</priority>
-        <example>
+		<priority>3</priority>
+		<example>
 <![CDATA[
 public class Foo {
     void good() {
@@ -38,7 +37,7 @@ public class Foo {
 }
 ]]>
         </example>
-    </rule>
-    
+	</rule>
+
 
 </ruleset>
\ No newline at end of file

From d7ac0b818657a3eca05f5c59de80eba42de2c42a Mon Sep 17 00:00:00 2001
From: Sergey Gorbaty <sgorbaty@apple.com>
Date: Wed, 18 Apr 2018 15:20:26 -0700
Subject: [PATCH 28/53] Using narrower scoped AST node + scope for global
 declarations

---
 .../rule/security/InsecureCryptoIvRule.java   | 39 ++++++++-----------
 .../rule/security/xml/InsecureCryptoIv.xml    |  9 +++--
 2 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
index 95e9593b84..aabec42e23 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java
@@ -6,21 +6,24 @@ package net.sourceforge.pmd.lang.java.rule.security;
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import net.sourceforge.pmd.lang.java.ast.ASTAllocationExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTArrayInitializer;
-import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceDeclaration;
+import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceBodyDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceType;
-import net.sourceforge.pmd.lang.java.ast.ASTFieldDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTLiteral;
 import net.sourceforge.pmd.lang.java.ast.ASTLocalVariableDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTName;
 import net.sourceforge.pmd.lang.java.ast.ASTPrimaryExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTPrimitiveType;
 import net.sourceforge.pmd.lang.java.ast.ASTReferenceType;
+import net.sourceforge.pmd.lang.java.ast.ASTVariableDeclarator;
 import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer;
 import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
+import net.sourceforge.pmd.lang.java.symboltable.VariableNameDeclaration;
+import net.sourceforge.pmd.lang.symboltable.NameOccurrence;
 
 /**
  * Finds hardcoded static Initialization Vectors vectors used with cryptographic
@@ -39,21 +42,14 @@ import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
 public class InsecureCryptoIvRule extends AbstractJavaRule {
 
     public InsecureCryptoIvRule() {
-        addRuleChainVisit(ASTClassOrInterfaceDeclaration.class);
+        addRuleChainVisit(ASTClassOrInterfaceBodyDeclaration.class);
     }
 
     @Override
-    public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
-        Set<ASTFieldDeclaration> foundFields = new HashSet<>();
+    public Object visit(ASTClassOrInterfaceBodyDeclaration node, Object data) {
         Set<ASTLocalVariableDeclaration> foundLocalVars = new HashSet<>();
         Set<String> passedInIvVarNames = new HashSet<>();
 
-        // byte[] fields
-        List<ASTFieldDeclaration> fields = node.findDescendantsOfType(ASTFieldDeclaration.class);
-        for (ASTFieldDeclaration field : fields) {
-            foundFields.addAll(extractPrimitiveTypes(field));
-        }
-
         // find new javax.crypto.spec.IvParameterSpec(...)
         List<ASTAllocationExpression> allocations = node.findDescendantsOfType(ASTAllocationExpression.class);
         for (ASTAllocationExpression allocation : allocations) {
@@ -78,9 +74,15 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
             foundLocalVars.addAll(extractPrimitiveTypes(localVar));
         }
 
-        for (ASTFieldDeclaration foundField : foundFields) {
-            if (passedInIvVarNames.contains(foundField.getVariableName())) {
-                validateProperIv(data, foundField.getFirstDescendantOfType(ASTVariableInitializer.class));
+        Map<VariableNameDeclaration, List<NameOccurrence>> globalDecls = node.getScope()
+                .getDeclarations(VariableNameDeclaration.class);
+
+        for (VariableNameDeclaration fieldVar : globalDecls.keySet()) {
+            if (passedInIvVarNames.contains(fieldVar.getNode().getImage())) {
+                ASTVariableDeclarator var = fieldVar.getNode().getFirstParentOfType(ASTVariableDeclarator.class);
+                if (var != null) {
+                    validateProperIv(data, var.getFirstDescendantOfType(ASTVariableInitializer.class));
+                }
             }
         }
 
@@ -101,15 +103,6 @@ public class InsecureCryptoIvRule extends AbstractJavaRule {
         return retVal;
     }
 
-    private Set<ASTFieldDeclaration> extractPrimitiveTypes(ASTFieldDeclaration field) {
-        List<ASTPrimitiveType> types = field.findDescendantsOfType(ASTPrimitiveType.class);
-        Set<ASTFieldDeclaration> retVal = new HashSet<>();
-
-        extractPrimitiveTypesInner(retVal, field, types);
-
-        return retVal;
-    }
-
     private <T> void extractPrimitiveTypesInner(Set<T> retVal, T field, List<ASTPrimitiveType> types) {
         for (ASTPrimitiveType type : types) {
             if (type.hasImageEqualTo("byte")) {
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
index 5cfa996a0b..6482224bef 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/security/xml/InsecureCryptoIv.xml
@@ -12,16 +12,20 @@ Hard coded inline IvSpec, bad
  import javax.crypto.Cipher;   
  
  public class Foo {
+ 
+ 	void outOfScope() {
+ 		byte[] ivBytes = new byte[16];  	 	
+ 	}
+    byte[] ivBytes = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, };  	
 
  	void encrypt() {
-		byte[] ivBytes = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; 
 		Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
 		cipher.init(Cipher.ENCRYPT_MODE, keySpec, new IvParameterSpec(ivBytes));		 
 	}
 }
      ]]></code>
 	</test-code>
-    <test-code>
+	<test-code>
 		<description><![CDATA[
 Randomly generated IV, good 
      ]]></description>
@@ -133,5 +137,4 @@ public class Foo {
      ]]></code>
 	</test-code>
 
-
 </test-data>

From fc2e0a625fb7f45c5ac8d600da32c07ee6300214 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sat, 21 Apr 2018 20:33:30 -0300
Subject: [PATCH 29/53] [python] Support comment based CPD suppressions

---
 pmd-python/etc/grammar/python.jj              |  6 +++++-
 .../sourceforge/pmd/cpd/PythonTokenizer.java  | 13 +++++++------
 .../pmd/cpd/PythonTokenizerTest.java          | 19 +++++++++++++++++++
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/pmd-python/etc/grammar/python.jj b/pmd-python/etc/grammar/python.jj
index e69d5886bd..60f723870c 100644
--- a/pmd-python/etc/grammar/python.jj
+++ b/pmd-python/etc/grammar/python.jj
@@ -33,7 +33,11 @@ SKIP :
 |   "\014"
 |   <CONTINUATION: ("\\") ("\r\n"|"\n"|"\r")>
 |   <NEWLINE: ("\r\n"|"\n"|"\r")>
-|   <TRAILING_COMMENT: "#" (~["\n","\r"])* >
+}
+
+SPECIAL_TOKEN :
+{
+    <TRAILING_COMMENT: "#" (~["\n","\r"])* >
 }
 
 TOKEN : /* SEPARATORS */
diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
index 9ab7965b0b..1783278024 100644
--- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
+++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java
@@ -9,9 +9,10 @@ import java.io.StringReader;
 
 import org.apache.commons.io.IOUtils;
 
+import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
+import net.sourceforge.pmd.cpd.token.TokenFilter;
 import net.sourceforge.pmd.lang.LanguageRegistry;
 import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
 import net.sourceforge.pmd.lang.ast.TokenMgrError;
 import net.sourceforge.pmd.lang.python.PythonLanguageModule;
 import net.sourceforge.pmd.lang.python.ast.Token;
@@ -31,13 +32,13 @@ public class PythonTokenizer implements Tokenizer {
                     .getDefaultVersion().getLanguageVersionHandler();
             reader = new StringReader(buffer.toString());
             reader = IOUtil.skipBOM(reader);
-            TokenManager tokenManager = languageVersionHandler
+            TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler
                     .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
+                    .getTokenManager(sourceCode.getFileName(), reader));
+            Token currentToken = (Token) tokenFilter.getNextToken();
+            while (currentToken != null) {
                 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
+                currentToken = (Token) tokenFilter.getNextToken();
             }
             tokenEntries.add(TokenEntry.getEOF());
             System.err.println("Added " + sourceCode);
diff --git a/pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java b/pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java
index 16798c6e64..eaa8cc1154 100644
--- a/pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java
+++ b/pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java
@@ -4,12 +4,15 @@
 
 package net.sourceforge.pmd.cpd;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
 
 public class PythonTokenizerTest extends AbstractTokenizerTest {
@@ -33,4 +36,20 @@ public class PythonTokenizerTest extends AbstractTokenizerTest {
         this.expectedTokenCount = 1218;
         super.tokenizeTest();
     }
+
+    @Test
+    public void testIgnoreBetweenSpecialComments() throws IOException {
+        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("import logging" + PMD.EOL
+                + "# CPD-OFF" + PMD.EOL
+                + "logger = logging.getLogger('django.request')" + PMD.EOL
+                + "class BaseHandler(object):" + PMD.EOL
+                + "    def __init__(self):" + PMD.EOL
+                + "        self._request_middleware = None" + PMD.EOL
+                + "        # CPD-ON" + PMD.EOL
+        ));
+        Tokens tokens = new Tokens();
+        tokenizer.tokenize(sourceCode, tokens);
+        TokenEntry.getEOF();
+        assertEquals(3, tokens.size()); // 3 tokens: "import" + "logging" + EOF
+    }
 }

From 6c19d9bed67eab3707471acf9450941a676fadfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 22 Apr 2018 02:48:35 -0300
Subject: [PATCH 30/53] Update changelog, refs #1039

---
 docs/pages/release_notes.md | 41 +++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 4fe0ddc0d3..55d5f4c968 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -15,6 +15,7 @@ This is a minor release.
 * [New and noteworthy](#new-and-noteworthy)
     *   [Tree transversal revision](#tree-transversal-revision)
     *   [Naming rules enhancements](#naming-rules-enhancements)
+    *   [CPD Suppression](#cpd-suppression)
     *   [Modified Rules](#modified-rules)
 * [Fixed Issues](#fixed-issues)
 * [API Changes](#api-changes)
@@ -44,6 +45,46 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
      using a regex property. See the rule's documentation for more info about
      configuration and default conventions.
 
+#### CPD Suppression
+
+Back in PMD 5.6.0 we introduced the ability to suppress CPD warnings in Java using comments, by
+including `CPD-OFF` (to start ignoring code), or `CPD-ON` (to resume analysis) during CPD execution.
+This has proved to be much more flexible and versatile than the old annotation-based approach,
+and has since been the prefered way to suppress CPD warnings.
+
+On this ocassion, we are xtending support for comment-based suppressions to many other languages:
+
+*   C/C++
+*   Ecmascript / Javascript
+*   Matlab
+*   Objective-C
+*   PL/SQL
+*   Python
+
+So for instance, in Python we could now do:
+
+```python
+class BaseHandler(object):
+    def __init__(self):
+        # some unignored code
+
+        # tell cpd to start ignoring code - CPD-OFF
+
+        # mission critical code, manually loop unroll
+        GoDoSomethingAwesome(x + x / 2);
+        GoDoSomethingAwesome(x + x / 2);
+        GoDoSomethingAwesome(x + x / 2);
+        GoDoSomethingAwesome(x + x / 2);
+        GoDoSomethingAwesome(x + x / 2);
+        GoDoSomethingAwesome(x + x / 2);
+
+        # resume CPD analysis - CPD-ON
+
+        # further code will *not* be ignored
+```
+
+Other languages are equivalent.
+
 #### Modified Rules
 
 *   The Java rule `UnnecessaryConstructor` (`java-codestyle`) has been rewritten as a Java rule (previously it was

From 933235b1409beb5199327e25aea78c65cc5c38c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= <clement.fournier76@gmail.com>
Date: Sun, 22 Apr 2018 18:05:30 +0200
Subject: [PATCH 31/53] Update release notes, refs #1041

Closes #1020
---
 docs/pages/release_notes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index bbdf01016a..f1c6ab85a9 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -61,6 +61,7 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 *   java
     *   [#894](https://github.com/pmd/pmd/issues/894): \[java] Maven PMD plugin fails to process some files without any explanation
     *   [#899](https://github.com/pmd/pmd/issues/899): \[java] JavaTypeDefinitionSimple.toString can cause NPEs
+    *   [#1020](https://github.com/pmd/pmd/issues/1020): \[java] The CyclomaticComplexity rule runs forever in 6.2.0
     *   [#1030](https://github.com/pmd/pmd/pull/1030): \[java] NoClassDefFoundError when analyzing PMD with PMD
 *   java-bestpractices
     *   [#370](https://github.com/pmd/pmd/issues/370): \[java] GuardLogStatementJavaUtil not considering lambdas
@@ -88,4 +89,5 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 *   [#1010](https://github.com/pmd/pmd/pull/1010): \[java] UnnecessaryConstructor triggered on required empty constructor (Dagger @Inject) - [BBG](https://github.com/djydewang)
 *   [#1012](https://github.com/pmd/pmd/pull/1012): \[java] JUnitAssertionsShouldIncludeMessage - False positive with assertEquals and JUnit5 - [BBG](https://github.com/djydewang)
 *   [#1024](https://github.com/pmd/pmd/pull/1024): \[java]Issue 558: Properlogger for enums - [Utku Cuhadaroglu](https://github.com/utkuc)
+*   [#1041](https://github.com/pmd/pmd/pull/1041): \[java] Make BasicProjectMemoizer thread safe. - [bergander](https://github.com/bergander)
 

From ce264cf0e45b7fbf86544f134c34a927b032f124 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Sun, 22 Apr 2018 16:00:28 -0300
Subject: [PATCH 32/53] Fix typos

---
 docs/pages/release_notes.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 55d5f4c968..9189a63c79 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -50,9 +50,9 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 Back in PMD 5.6.0 we introduced the ability to suppress CPD warnings in Java using comments, by
 including `CPD-OFF` (to start ignoring code), or `CPD-ON` (to resume analysis) during CPD execution.
 This has proved to be much more flexible and versatile than the old annotation-based approach,
-and has since been the prefered way to suppress CPD warnings.
+and has since been the preferred way to suppress CPD warnings.
 
-On this ocassion, we are xtending support for comment-based suppressions to many other languages:
+On this ocassion, we are extending support for comment-based suppressions to many other languages:
 
 *   C/C++
 *   Ecmascript / Javascript

From 1a51582963469b8af6e63dcd83bc175e33681fb3 Mon Sep 17 00:00:00 2001
From: Akshat Bahety <akshatbahety@gmail.com>
Date: Mon, 23 Apr 2018 02:08:16 +0530
Subject: [PATCH 33/53] Solution for issue #816

Fixed the issue with traversing the tree and checking out if the getInstance method is used twice under the same class.
---
 .../errorprone/SingleMethodSingletonRule.java | 57 +++++++++++++------
 1 file changed, 40 insertions(+), 17 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
index ead66d667a..a47f6890b0 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
@@ -4,35 +4,58 @@
 
 package net.sourceforge.pmd.lang.java.rule.errorprone;
 
-import java.util.HashSet;
-import java.util.Set;
 
-import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit;
+import java.util.List;
+
+
+
+import net.sourceforge.pmd.lang.java.ast.ASTClassOrInterfaceDeclaration;
 import net.sourceforge.pmd.lang.java.ast.ASTMethodDeclaration;
 import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
 
+/**
+ * Returns Checks if the singleton rule is used properly.
+ */
 public class SingleMethodSingletonRule extends AbstractJavaRule {
 
-    private Set<String> methodset = new HashSet<String>();
+    /**
+     * Checks for getInstance method usage in the same class.
+     * @param node of ASTCLass
+     * @param data of Object
+     * @return Object
+     *
+     */
 
-    @Override
-    public Object visit(ASTCompilationUnit node, Object data) {
-        methodset.clear();
-        return super.visit(node, data);
-    }
 
-    @Override
-    public Object visit(ASTMethodDeclaration node, Object data) {
-        if (node.getResultType().isVoid()) {
-            return super.visit(node, data);
-        }
+    public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
 
-        if ("getInstance".equals(node.getMethodName())) {
-            if (!methodset.add(node.getMethodName())) {
-                addViolation(data, node);
+        String a = node.getImage(); // Get the name of the Class it's part of
+        System.out.println(a);
+
+        List<ASTMethodDeclaration> methods = node.findDescendantsOfType(ASTMethodDeclaration.class); // Find the name of methods in it
+
+        System.out.println(methods);
+
+        int count = 0;
+        for (ASTMethodDeclaration method : methods) {
+
+            System.out.println(method.getName());
+            if (method.getName().equals("getInstance")) {
+                count++;
             }
+
         }
 
+        if (count > 1) {
+            System.out.println("error");
+            addViolation(data, node);
+        }
+
+        /*
+        Can now check if each class has only one getInstance methods than it's all sorted.
+        */
+
         return super.visit(node, data);
+
     }
 }

From a91bc4bfb6582aabf3b3d991931e52817b43b72f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Mon, 23 Apr 2018 01:02:35 -0300
Subject: [PATCH 34/53] [core] Log only once all messages

 - CLI runs log only to stderr, never to stdout
 - Both Ant and CLI runs log only once, the root logger never gets a
hold of the logs, completely isolating PMD
---
 pmd-core/src/main/java/net/sourceforge/pmd/PMD.java         | 6 ++----
 .../net/sourceforge/pmd/util/log/ConsoleLogHandler.java     | 2 ++
 .../sourceforge/pmd/util/log/ScopedLogHandlersManager.java  | 3 +++
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java
index adb01dc245..9555c86d21 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java
@@ -15,7 +15,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Handler;
+import java.util.logging.ConsoleHandler;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -45,7 +45,6 @@ import net.sourceforge.pmd.util.database.DBURI;
 import net.sourceforge.pmd.util.database.SourceObject;
 import net.sourceforge.pmd.util.datasource.DataSource;
 import net.sourceforge.pmd.util.datasource.ReaderDataSource;
-import net.sourceforge.pmd.util.log.ConsoleLogHandler;
 import net.sourceforge.pmd.util.log.ScopedLogHandlersManager;
 
 /**
@@ -450,8 +449,7 @@ public class PMD {
         final PMDConfiguration configuration = params.toConfiguration();
 
         final Level logLevel = params.isDebug() ? Level.FINER : Level.INFO;
-        final Handler logHandler = new ConsoleLogHandler();
-        final ScopedLogHandlersManager logHandlerManager = new ScopedLogHandlersManager(logLevel, logHandler);
+        final ScopedLogHandlersManager logHandlerManager = new ScopedLogHandlersManager(logLevel, new ConsoleHandler());
         final Level oldLogLevel = LOG.getLevel();
         // Need to do this, since the static logger has already been initialized
         // at this point
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ConsoleLogHandler.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ConsoleLogHandler.java
index 42c0389f90..8e61d3d27b 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ConsoleLogHandler.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ConsoleLogHandler.java
@@ -14,7 +14,9 @@ import java.util.logging.LogRecord;
  * Log to the console using a basic formatter.
  *
  * @author Wouter Zelle
+ * @deprecated This class will be complety removed in 7.0.0
  */
+@Deprecated
 public class ConsoleLogHandler extends Handler {
 
     private static final Formatter FORMATTER = new PmdLogFormatter();
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ScopedLogHandlersManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ScopedLogHandlersManager.java
index c6e26b6dda..53cb824944 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ScopedLogHandlersManager.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/log/ScopedLogHandlersManager.java
@@ -36,7 +36,9 @@ public class ScopedLogHandlersManager {
         }
         for (Handler handler : newHandlers) {
             logger.addHandler(handler);
+            handler.setLevel(level);
         }
+        logger.setUseParentHandlers(false);
     }
 
     public void close() {
@@ -47,5 +49,6 @@ public class ScopedLogHandlersManager {
             logger.addHandler(handler);
         }
         logger.setLevel(oldLogLevel);
+        logger.setUseParentHandlers(true);
     }
 }

From 59bfc7ab52cb4f8ef1809f4c2a341dae8ca7e799 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Mon, 23 Apr 2018 01:05:34 -0300
Subject: [PATCH 35/53] Update changelog, refs #1045

---
 docs/pages/release_notes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index f1c6ab85a9..c2ac8d8110 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -56,6 +56,7 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 
 *   all
     *   [#988](https://github.com/pmd/pmd/issues/988): \[core] FileNotFoundException for missing classes directory with analysis cache enabled
+    *   [#1036](https://github.com/pmd/pmd/issues/1036): \[core] Non-XML output breaks XML-based CLI integrations
 *   documentation
     *   [#994](https://github.com/pmd/pmd/issues/994): \[doc] Delete duplicate page contributing.md on the website
 *   java

From bee7adb33a077bbd52fd31fc66c5ad78285f7ee8 Mon Sep 17 00:00:00 2001
From: Akshat Bahety <akshatbahety@gmail.com>
Date: Mon, 23 Apr 2018 09:48:34 +0530
Subject: [PATCH 36/53] Removed Debugging parts

Updated the file, removed the unnecessary debugging statements
---
 .../errorprone/SingleMethodSingletonRule.java | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
index a47f6890b0..293ecea630 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
@@ -29,31 +29,22 @@ public class SingleMethodSingletonRule extends AbstractJavaRule {
 
     public Object visit(ASTClassOrInterfaceDeclaration node, Object data) {
 
-        String a = node.getImage(); // Get the name of the Class it's part of
-        System.out.println(a);
 
         List<ASTMethodDeclaration> methods = node.findDescendantsOfType(ASTMethodDeclaration.class); // Find the name of methods in it
 
-        System.out.println(methods);
-
         int count = 0;
         for (ASTMethodDeclaration method : methods) {
-
-            System.out.println(method.getName());
+            
             if (method.getName().equals("getInstance")) {
                 count++;
+                if(count > 1){
+                   addViolation(data, node); 
+                    break;
+                }
             }
 
         }
 
-        if (count > 1) {
-            System.out.println("error");
-            addViolation(data, node);
-        }
-
-        /*
-        Can now check if each class has only one getInstance methods than it's all sorted.
-        */
 
         return super.visit(node, data);
 

From c137ebaa60c0121ad6e82f96db86f3498f46366e Mon Sep 17 00:00:00 2001
From: Akshat Bahety <akshatbahety@gmail.com>
Date: Mon, 23 Apr 2018 10:04:32 +0530
Subject: [PATCH 37/53] CheckStyle Updated

---
 .../java/rule/errorprone/SingleMethodSingletonRule.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
index 293ecea630..8336ba8c0d 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/SingleMethodSingletonRule.java
@@ -34,11 +34,11 @@ public class SingleMethodSingletonRule extends AbstractJavaRule {
 
         int count = 0;
         for (ASTMethodDeclaration method : methods) {
-            
+
             if (method.getName().equals("getInstance")) {
                 count++;
-                if(count > 1){
-                   addViolation(data, node); 
+                if (count > 1) {
+                    addViolation(data, node);
                     break;
                 }
             }

From 92ecdde2f4448c0ea4541c76baf64e586eb892f0 Mon Sep 17 00:00:00 2001
From: Akshat Bahety <akshatbahety@gmail.com>
Date: Mon, 23 Apr 2018 10:29:43 +0530
Subject: [PATCH 38/53] Singleton Test Case

Added a test case to check the singleton rule for nested classes
---
 .../errorprone/xml/SingleMethodSingleton.xml  | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/SingleMethodSingleton.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/SingleMethodSingleton.xml
index a04d3e2800..ea2cc5b43d 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/SingleMethodSingleton.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/SingleMethodSingleton.xml
@@ -48,4 +48,27 @@ private static Singleton instance = null;
 ]]>
 		</code>
 	</test-code>
-</test-data>
\ No newline at end of file
+
+    <test-code>
+        <description><![CDATA[
+OK! Has two different getInstance() in different classes
+     ]]></description>
+        <expected-problems>0</expected-problems>
+        <code>
+            <![CDATA[
+final class Siblings {
+  static class Inner1 {
+    static Inner1 getInstance() { return null; }
+  }
+  static class Inner2 {
+    static Inner2 getInstance() { return null; }
+  }
+}
+]]>
+        </code>
+    </test-code>
+
+
+
+
+</test-data>

From dac6d8c1a59262bb2103bf206b03dcda0578efe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Mon, 23 Apr 2018 02:15:45 -0300
Subject: [PATCH 39/53] Update changelog, refs #1044

---
 docs/pages/release_notes.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index f1c6ab85a9..40f41e4d2e 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -71,6 +71,8 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 *   java-codestyle
     *   [#1003](https://github.com/pmd/pmd/issues/1003): \[java] UnnecessaryConstructor triggered on required empty constructor (Dagger @Inject)
     *   [#1023](https://github.com/pmd/pmd/issues/1023): \[java] False positive for useless parenthesis
+*   java-errorprone
+    *   [#816](https://github.com/pmd/pmd/issues/816): \[java] SingleMethodSingleton false positives with inner classes
 *   java-performance
     *   [#586](https://github.com/pmd/pmd/issues/586): \[java] AvoidUsingShortType erroneously triggered on overrides of 3rd party methods
 
@@ -90,4 +92,5 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 *   [#1012](https://github.com/pmd/pmd/pull/1012): \[java] JUnitAssertionsShouldIncludeMessage - False positive with assertEquals and JUnit5 - [BBG](https://github.com/djydewang)
 *   [#1024](https://github.com/pmd/pmd/pull/1024): \[java]Issue 558: Properlogger for enums - [Utku Cuhadaroglu](https://github.com/utkuc)
 *   [#1041](https://github.com/pmd/pmd/pull/1041): \[java] Make BasicProjectMemoizer thread safe. - [bergander](https://github.com/bergander)
+*   [#1044](https://github.com/pmd/pmd/pull/1044): \[java] Fix for issue #816 - [Akshat Bahety](https://github.com/akshatbahety)
 

From f855b572cc8dbf24199d0277d0b59b637507b897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Mon, 23 Apr 2018 03:09:20 -0300
Subject: [PATCH 40/53] Update userdocs

---
 docs/pages/pmd/userdocs/cpd.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/pages/pmd/userdocs/cpd.md b/docs/pages/pmd/userdocs/cpd.md
index 747bcf6f6b..c3690ee0d8 100644
--- a/docs/pages/pmd/userdocs/cpd.md
+++ b/docs/pages/pmd/userdocs/cpd.md
@@ -438,7 +438,8 @@ Here's a screenshot of CPD after running on the JDK 8 java.lang package:
 
 ## Suppression
 
-Arbitrary blocks of code can be ignored through comments on **Java** by including the keywords `CPD-OFF` and `CPD-ON`.
+Arbitrary blocks of code can be ignored through comments on **Java**, **C/C++**, **Javascript**, **Matlab**,
+**Objective-C**, **PL/SQL** and **Python** by including the keywords `CPD-OFF` and `CPD-ON`.
 
     public Object someParameterizedFactoryMethod(int x) throws Exception {
         // some unignored code

From 71ebcca07947c4ba970e3e3f4ec7b70be224ba2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= <clement.fournier76@gmail.com>
Date: Mon, 23 Apr 2018 14:28:13 +0200
Subject: [PATCH 41/53] Fix typos in release notes

---
 docs/pages/release_notes.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 40f41e4d2e..1d180b12b0 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -13,7 +13,7 @@ This is a minor release.
 ### Table Of Contents
 
 * [New and noteworthy](#new-and-noteworthy)
-    *   [Tree transversal revision](#tree-transversal-revision)
+    *   [Tree traversal revision](#tree-traversal-revision)
     *   [Naming rules enhancements](#naming-rules-enhancements)
     *   [Modified Rules](#modified-rules)
 * [Fixed Issues](#fixed-issues)
@@ -23,7 +23,7 @@ This is a minor release.
 
 ### New and noteworthy
 
-#### Tree transversal revision
+#### Tree traversal revision
 
 As described in [#904](https://github.com/pmd/pmd/issues/904), when searching for child nodes of the AST methods
 such as `hasDescendantOfType`, `getFirstDescendantOfType` and `findDescendantsOfType` were found to behave inconsistently,
@@ -33,7 +33,7 @@ find boundaries.
 
 This change implies several false positives / unexpected results (ie: `ASTBlockStatement` falsely returning `true` to `isAllocation()`)
 have been fixed; and lots of searches are now restricted to smaller search areas, which improves performance (depending on the project,
-we have measured up to 10% improvements during Type Resolution, Symbol Table analysis, and some rule's application).
+we have measured up to 10% improvements during Type Resolution, Symbol Table analysis, and some rules' application).
 
 #### Naming rules enhancements
 

From 43e9a7c31bb8b513b2ee2d41a4128802c6686413 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Mon, 23 Apr 2018 18:57:46 +0200
Subject: [PATCH 42/53] Update release notes, fixes #695

---
 docs/pages/release_notes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 9189a63c79..ea83c39b1a 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -96,6 +96,7 @@ Other languages are equivalent.
 ### Fixed Issues
 
 *   all
+    *   [#695](https://github.com/pmd/pmd/issues/695): \[core] Extend comment-based suppression to all JavaCC languages
     *   [#988](https://github.com/pmd/pmd/issues/988): \[core] FileNotFoundException for missing classes directory with analysis cache enabled
 *   documentation
     *   [#994](https://github.com/pmd/pmd/issues/994): \[doc] Delete duplicate page contributing.md on the website

From bc9b18cc6998aebe315e98de51c507aa71edb342 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Tue, 10 Apr 2018 17:07:19 +0200
Subject: [PATCH 43/53] [java] NullAssignment false positive Fixes #629

---
 docs/pages/release_notes.md                   |  1 +
 .../rule/errorprone/NullAssignmentRule.java   | 23 ++++--
 .../rule/errorprone/xml/NullAssignment.xml    | 71 +++++++++++++++++--
 3 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 7bccb2fbbe..3fcce04aea 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -114,6 +114,7 @@ Other languages are equivalent.
     *   [#1003](https://github.com/pmd/pmd/issues/1003): \[java] UnnecessaryConstructor triggered on required empty constructor (Dagger @Inject)
     *   [#1023](https://github.com/pmd/pmd/issues/1023): \[java] False positive for useless parenthesis
 *   java-errorprone
+    *   [#629](https://github.com/pmd/pmd/issues/629): \[java] NullAssignment false positive
     *   [#816](https://github.com/pmd/pmd/issues/816): \[java] SingleMethodSingleton false positives with inner classes
 *   java-performance
     *   [#586](https://github.com/pmd/pmd/issues/586): \[java] AvoidUsingShortType erroneously triggered on overrides of 3rd party methods
diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
index 9cbf691a06..195300d7f5 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
@@ -5,21 +5,25 @@
 package net.sourceforge.pmd.lang.java.rule.errorprone;
 
 import net.sourceforge.pmd.lang.java.ast.ASTAssignmentOperator;
+import net.sourceforge.pmd.lang.java.ast.ASTBlockStatement;
 import net.sourceforge.pmd.lang.java.ast.ASTConditionalExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTEqualityExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTName;
 import net.sourceforge.pmd.lang.java.ast.ASTNullLiteral;
+import net.sourceforge.pmd.lang.java.ast.ASTReturnStatement;
 import net.sourceforge.pmd.lang.java.ast.ASTStatementExpression;
+import net.sourceforge.pmd.lang.java.ast.ASTVariableInitializer;
 import net.sourceforge.pmd.lang.java.ast.AccessNode;
 import net.sourceforge.pmd.lang.java.rule.AbstractJavaRule;
 import net.sourceforge.pmd.lang.java.symboltable.VariableNameDeclaration;
 
-// TODO - should check that this is not the first assignment.  e.g., this is OK:
-// Object x;
-// x = null;
 public class NullAssignmentRule extends AbstractJavaRule {
 
+    public NullAssignmentRule() {
+        addRuleChainVisit(ASTNullLiteral.class);
+    }
+
     @Override
     public Object visit(ASTNullLiteral node, Object data) {
 
@@ -56,6 +60,17 @@ public class NullAssignmentRule extends AbstractJavaRule {
     }
 
     private boolean isBadTernary(ASTConditionalExpression n) {
-        return n.isTernary() && !(n.jjtGetChild(0) instanceof ASTEqualityExpression);
+        boolean isInitializer = false;
+
+        ASTVariableInitializer variableInitializer = n.getFirstParentOfType(ASTVariableInitializer.class);
+        if (variableInitializer != null) {
+            ASTBlockStatement statement = n.getFirstParentOfType(ASTBlockStatement.class);
+            isInitializer = statement == variableInitializer.getFirstParentOfType(ASTBlockStatement.class);
+        }
+
+        return n.isTernary()
+                && !(n.jjtGetChild(0) instanceof ASTEqualityExpression)
+                && !isInitializer
+                && !(n.getNthParent(2) instanceof ASTReturnStatement);
     }
 }
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
index bc183811ac..b3a1273fce 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
@@ -64,10 +64,8 @@ public class Foo {
      ]]></code>
     </test-code>
     <test-code>
-        <description><![CDATA[
-null assignment in ternary
-     ]]></description>
-        <expected-problems>1</expected-problems>
+        <description>null assignment in ternary - initialization</description>
+        <expected-problems>0</expected-problems>
         <code><![CDATA[
 public class Foo {
  public void foo() {
@@ -77,15 +75,37 @@ public class Foo {
      ]]></code>
     </test-code>
     <test-code>
-        <description><![CDATA[
-null assignment in ternary, part deux
-     ]]></description>
+        <description>null assignment in ternary</description>
         <expected-problems>1</expected-problems>
         <code><![CDATA[
+public class Foo {
+ public void foo() {
+  String x;
+  x = bar() ? "fiz" : null;
+ }
+}
+     ]]></code>
+    </test-code>
+    <test-code>
+        <description>null assignment in ternary, part deux - initialization</description>
+        <expected-problems>0</expected-problems>
+        <code><![CDATA[
 public class Foo {
  public void foo() {
   String x = bar() ? null : "fiz";
  }
+}
+     ]]></code>
+    </test-code>
+    <test-code>
+        <description>null assignment in ternary, part deux</description>
+        <expected-problems>1</expected-problems>
+        <code><![CDATA[
+public class Foo {
+ public void foo() {
+  String x;
+  x = bar() ? null : "fiz";
+ }
 }
      ]]></code>
     </test-code>
@@ -129,4 +149,41 @@ public class Foo {
 }
      ]]></code>
     </test-code>
+
+    <test-code>
+        <description>[java] NullAssignment false positive - initialization #629</description>
+        <expected-problems>0</expected-problems>
+        <code><![CDATA[
+public class NullAssignmentFP {
+    public void foo() {
+        Type result = someHash.computeIfAbsent(a, _unused -> test ? truthy : null);
+    }
+}
+        ]]></code>
+    </test-code>
+
+    <test-code>
+        <description>[java] NullAssignment false positive - actual assignment #629</description>
+        <expected-problems>1</expected-problems>
+        <code><![CDATA[
+public class NullAssignmentFP {
+    public void foo() {
+        Type result;
+        result = someHash.computeIfAbsent(a, _unused -> test ? truthy : null);
+    }
+}
+        ]]></code>
+    </test-code>
+
+    <test-code>
+        <description>[java] NullAssignment false positive - return with ternary #629</description>
+        <expected-problems>0</expected-problems>
+        <code><![CDATA[
+public class NullAssignmentFP {
+    public DateTime foo(DateTime dateTime) {
+        return dateTime.getYear() < 2100 ? dateTime : null;
+    }
+}
+        ]]></code>
+    </test-code>
 </test-data>

From fd71799c8310f28c69e4eda30241322a298c1b4e Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Mon, 23 Apr 2018 19:30:58 +0200
Subject: [PATCH 44/53] Don't consider null literal inside lambda

---
 .../java/rule/errorprone/NullAssignmentRule.java   | 14 ++++++++------
 .../java/rule/errorprone/xml/NullAssignment.xml    |  4 ++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
index 195300d7f5..668e98eca4 100644
--- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
+++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/NullAssignmentRule.java
@@ -9,6 +9,7 @@ import net.sourceforge.pmd.lang.java.ast.ASTBlockStatement;
 import net.sourceforge.pmd.lang.java.ast.ASTConditionalExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTEqualityExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTExpression;
+import net.sourceforge.pmd.lang.java.ast.ASTLambdaExpression;
 import net.sourceforge.pmd.lang.java.ast.ASTName;
 import net.sourceforge.pmd.lang.java.ast.ASTNullLiteral;
 import net.sourceforge.pmd.lang.java.ast.ASTReturnStatement;
@@ -59,18 +60,19 @@ public class NullAssignmentRule extends AbstractJavaRule {
                 && ((AccessNode) ((VariableNameDeclaration) name.getNameDeclaration()).getAccessNodeParent()).isFinal();
     }
 
-    private boolean isBadTernary(ASTConditionalExpression n) {
+    private boolean isBadTernary(ASTConditionalExpression ternary) {
         boolean isInitializer = false;
 
-        ASTVariableInitializer variableInitializer = n.getFirstParentOfType(ASTVariableInitializer.class);
+        ASTVariableInitializer variableInitializer = ternary.getFirstParentOfType(ASTVariableInitializer.class);
         if (variableInitializer != null) {
-            ASTBlockStatement statement = n.getFirstParentOfType(ASTBlockStatement.class);
+            ASTBlockStatement statement = ternary.getFirstParentOfType(ASTBlockStatement.class);
             isInitializer = statement == variableInitializer.getFirstParentOfType(ASTBlockStatement.class);
         }
 
-        return n.isTernary()
-                && !(n.jjtGetChild(0) instanceof ASTEqualityExpression)
+        return ternary.isTernary()
+                && !(ternary.jjtGetChild(0) instanceof ASTEqualityExpression)
                 && !isInitializer
-                && !(n.getNthParent(2) instanceof ASTReturnStatement);
+                && !(ternary.getNthParent(2) instanceof ASTReturnStatement)
+                && !(ternary.getNthParent(2) instanceof ASTLambdaExpression);
     }
 }
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
index b3a1273fce..ef656c0d50 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/NullAssignment.xml
@@ -163,8 +163,8 @@ public class NullAssignmentFP {
     </test-code>
 
     <test-code>
-        <description>[java] NullAssignment false positive - actual assignment #629</description>
-        <expected-problems>1</expected-problems>
+        <description>[java] NullAssignment false positive - no direct assignment, but lambda #629</description>
+        <expected-problems>0</expected-problems>
         <code><![CDATA[
 public class NullAssignmentFP {
     public void foo() {

From 391193c0443bdd75f4bc806841b0ac4a9c1e27cc Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Mon, 23 Apr 2018 20:29:13 +0200
Subject: [PATCH 45/53] PMD dogfooding - avoid unnecessary local before return

---
 .../java/net/sourceforge/pmd/cpd/CPDConfiguration.java     | 3 +--
 pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java    | 7 ++-----
 .../java/net/sourceforge/pmd/util/designer/Designer.java   | 6 ++----
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java
index dbca59a615..ad5d8cee58 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java
@@ -351,7 +351,7 @@ public class CPDConfiguration extends AbstractConfiguration {
             }
         }
 
-        FilenameFilter filter = new FilenameFilter() {
+        return new FilenameFilter() {
             @Override
             public boolean accept(File dir, String name) {
                 File f = new File(dir, name);
@@ -362,7 +362,6 @@ public class CPDConfiguration extends AbstractConfiguration {
                 return languageFilter.accept(dir, name);
             }
         };
-        return filter;
     }
 
     /**
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java
index 3f568c430f..c51770960f 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java
@@ -720,7 +720,7 @@ public class GUI implements CPDListener {
 
         final long start = System.currentTimeMillis();
 
-        Timer t = new Timer(1000, new ActionListener() {
+        return new Timer(1000, new ActionListener() {
             @Override
             public void actionPerformed(ActionEvent e) {
                 long now = System.currentTimeMillis();
@@ -731,7 +731,6 @@ public class GUI implements CPDListener {
                 timeField.setText(formatTime(minutes, seconds));
             }
         });
-        return t;
     }
 
     private static String formatTime(long minutes, long seconds) {
@@ -762,7 +761,7 @@ public class GUI implements CPDListener {
 
     private TableModel tableModelFrom(final List<Match> items) {
 
-        TableModel model = new SortingTableModel<Match>() {
+        return new SortingTableModel<Match>() {
 
             private int sortColumn;
             private boolean sortDescending;
@@ -837,8 +836,6 @@ public class GUI implements CPDListener {
                 }
             }
         };
-
-        return model;
     }
 
     private void sortOnColumn(int columnIndex) {
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/designer/Designer.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/designer/Designer.java
index a70f3fbc67..c82c81353d 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/util/designer/Designer.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/designer/Designer.java
@@ -312,7 +312,7 @@ public class Designer implements ClipboardOwner {
 
         @Override
         public Enumeration<TreeNode> children() {
-            Enumeration<TreeNode> e = new Enumeration<TreeNode>() {
+            return new Enumeration<TreeNode>() {
                 int i = 0;
 
                 @Override
@@ -325,7 +325,6 @@ public class Designer implements ClipboardOwner {
                     return kids[i++];
                 }
             };
-            return e;
         }
 
         @Override
@@ -395,7 +394,7 @@ public class Designer implements ClipboardOwner {
                 getChildAt(0); // force it to build kids
             }
 
-            Enumeration<TreeNode> e = new Enumeration<TreeNode>() {
+            return new Enumeration<TreeNode>() {
                 int i = 0;
 
                 @Override
@@ -408,7 +407,6 @@ public class Designer implements ClipboardOwner {
                     return kids[i++];
                 }
             };
-            return e;
         }
 
         @Override

From 1f662e81f7eb772148d0f831ed1f853475d69fff Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Mon, 23 Apr 2018 21:01:11 +0200
Subject: [PATCH 46/53] Update release notes, refs #1042 - new rule

---
 docs/pages/release_notes.md                   | 15 ++++++++---
 .../main/resources/rulesets/releases/630.xml  | 13 +++++++++
 .../main/resources/category/java/security.xml | 27 +++++++++----------
 3 files changed, 38 insertions(+), 17 deletions(-)
 create mode 100644 pmd-core/src/main/resources/rulesets/releases/630.xml

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index bbdf01016a..e022708404 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -15,6 +15,7 @@ This is a minor release.
 * [New and noteworthy](#new-and-noteworthy)
     *   [Tree transversal revision](#tree-transversal-revision)
     *   [Naming rules enhancements](#naming-rules-enhancements)
+    *   [New Rules](#new-rules)
     *   [Modified Rules](#modified-rules)
 * [Fixed Issues](#fixed-issues)
 * [API Changes](#api-changes)
@@ -44,10 +45,17 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
      using a regex property. See the rule's documentation for more info about
      configuration and default conventions.
 
+#### New Rules
+
+*   The new Java rule [`InsecureCryptoIv`](pmd_rules_java_security.html#insecurecryptoiv) (`java-security`)
+    detects hard coded initialization vectors used in cryptographic operations. It is recommended to use
+    a randomly generated IV.
+
 #### Modified Rules
 
-*   The Java rule `UnnecessaryConstructor` (`java-codestyle`) has been rewritten as a Java rule (previously it was
-    a XPath-based rule). It supports a new property `ignoredAnnotations` and ignores by default empty constructors,
+*   The Java rule [`UnnecessaryConstructor`](pmd_rules_java_codestyle.html#unnecessaryconstructor) (`java-codestyle`)
+    has been rewritten as a Java rule (previously it was a XPath-based rule). It supports a new property
+    `ignoredAnnotations` and ignores by default empty constructors,
     that are annotated with `javax.inject.Inject`. Additionally, it detects now also unnecessary private constructors
     in enums.
 
@@ -87,5 +95,6 @@ we have measured up to 10% improvements during Type Resolution, Symbol Table ana
 *   [#1008](https://github.com/pmd/pmd/pull/1008): \[core] DOC: fix closing tag for &lt;pmdVersion> - [stonio](https://github.com/stonio)
 *   [#1010](https://github.com/pmd/pmd/pull/1010): \[java] UnnecessaryConstructor triggered on required empty constructor (Dagger @Inject) - [BBG](https://github.com/djydewang)
 *   [#1012](https://github.com/pmd/pmd/pull/1012): \[java] JUnitAssertionsShouldIncludeMessage - False positive with assertEquals and JUnit5 - [BBG](https://github.com/djydewang)
-*   [#1024](https://github.com/pmd/pmd/pull/1024): \[java]Issue 558: Properlogger for enums - [Utku Cuhadaroglu](https://github.com/utkuc)
+*   [#1024](https://github.com/pmd/pmd/pull/1024): \[java] Issue 558: Properlogger for enums - [Utku Cuhadaroglu](https://github.com/utkuc)
+*   [#1042](https://github.com/pmd/pmd/pull/1042): \[java] New security rule: report usage of hard coded IV in crypto operations - [Sergey Gorbaty](https://github.com/sgorbaty)
 
diff --git a/pmd-core/src/main/resources/rulesets/releases/630.xml b/pmd-core/src/main/resources/rulesets/releases/630.xml
new file mode 100644
index 0000000000..a10a05506a
--- /dev/null
+++ b/pmd-core/src/main/resources/rulesets/releases/630.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+
+<ruleset name="630"
+    xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+  <description>
+This ruleset contains links to rules that are new in PMD v6.3.0
+  </description>
+
+    <rule ref="category/java/security.xml/InsecureCryptoIv"/>
+</ruleset>
+
diff --git a/pmd-java/src/main/resources/category/java/security.xml b/pmd-java/src/main/resources/category/java/security.xml
index d148f3cdee..e73df34fed 100644
--- a/pmd-java/src/main/resources/category/java/security.xml
+++ b/pmd-java/src/main/resources/category/java/security.xml
@@ -1,23 +1,23 @@
 <?xml version="1.0"?>
 
-<ruleset name="Security"
-	xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+<ruleset name="Security" xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
 
-	<description>
+    <description>
 Rules that flag potential security flaws.
     </description>
 
-	<rule name="InsecureCryptoIv" since="6.3.0"
-		message="Do not use hard coded initialization vector in crypto operations"
-		class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
-		externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
-		<description>
+    <rule name="InsecureCryptoIv"
+          since="6.3.0"
+          message="Do not use hard coded initialization vector in crypto operations"
+          class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
+          externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
+        <description>
 Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV.
         </description>
-		<priority>3</priority>
-		<example>
+        <priority>3</priority>
+        <example>
 <![CDATA[
 public class Foo {
     void good() {
@@ -37,7 +37,6 @@ public class Foo {
 }
 ]]>
         </example>
-	</rule>
-
+    </rule>
 
 </ruleset>
\ No newline at end of file

From 260f1da4402418e74e531edcc0db5de490364166 Mon Sep 17 00:00:00 2001
From: "Travis CI (pmd-bot)" <andreas.dangel+pmd-bot@adangel.org>
Date: Mon, 23 Apr 2018 19:20:29 +0000
Subject: [PATCH 47/53] Update documentation

---
 docs/_data/sidebars/pmd_sidebar.yml   |  3 ++
 docs/pages/pmd/rules/java.md          |  6 ++++
 docs/pages/pmd/rules/java/security.md | 46 +++++++++++++++++++++++++++
 3 files changed, 55 insertions(+)
 create mode 100644 docs/pages/pmd/rules/java/security.md

diff --git a/docs/_data/sidebars/pmd_sidebar.yml b/docs/_data/sidebars/pmd_sidebar.yml
index 1c722c3d9e..d8444b7bae 100644
--- a/docs/_data/sidebars/pmd_sidebar.yml
+++ b/docs/_data/sidebars/pmd_sidebar.yml
@@ -172,6 +172,9 @@ entries:
         - title: Performance
           output: web, pdf
           url: /pmd_rules_java_performance.html
+        - title: Security
+          output: web, pdf
+          url: /pmd_rules_java_security.html
     - title: null
       output: web, pdf
       subfolders:
diff --git a/docs/pages/pmd/rules/java.md b/docs/pages/pmd/rules/java.md
index 52035fb158..8e8eba921c 100644
--- a/docs/pages/pmd/rules/java.md
+++ b/docs/pages/pmd/rules/java.md
@@ -323,6 +323,12 @@ folder: pmd/rules
 *   [UseStringBufferForStringAppends](pmd_rules_java_performance.html#usestringbufferforstringappends): The use of the '+=' operator for appending strings causes the JVM to create and use an internal S...
 *   [UseStringBufferLength](pmd_rules_java_performance.html#usestringbufferlength): Use StringBuffer.length() to determine StringBuffer length rather than using StringBuffer.toStrin...
 
+## Security
+
+{% include callout.html content="Rules that flag potential security flaws." %}
+
+*   [InsecureCryptoIv](pmd_rules_java_security.html#insecurecryptoiv): Do not use hard coded initialization vector in cryptographic operations. Please use a randomly ge...
+
 ## Additional rulesets
 
 *   Android (`rulesets/java/android.xml`):
diff --git a/docs/pages/pmd/rules/java/security.md b/docs/pages/pmd/rules/java/security.md
new file mode 100644
index 0000000000..17b9caa150
--- /dev/null
+++ b/docs/pages/pmd/rules/java/security.md
@@ -0,0 +1,46 @@
+---
+title: Security
+summary: Rules that flag potential security flaws.
+permalink: pmd_rules_java_security.html
+folder: pmd/rules/java
+sidebaractiveurl: /pmd_rules_java.html
+editmepath: ../pmd-java/src/main/resources/category/java/security.xml
+keywords: Security, InsecureCryptoIv
+language: Java
+---
+## InsecureCryptoIv
+
+**Since:** PMD 6.3.0
+
+**Priority:** Medium (3)
+
+Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV.
+
+**This rule is defined by the following Java class:** [net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule](https://github.com/pmd/pmd/blob/master/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/InsecureCryptoIvRule.java)
+
+**Example(s):**
+
+``` java
+public class Foo {
+    void good() {
+        SecureRandom random = new SecureRandom();
+        byte iv[] = new byte[16];
+        random.nextBytes(bytes);
+    }
+
+    void bad() {
+        byte[] iv = new byte[] { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, };
+    }
+    
+    void alsoBad() {
+        byte[] iv = "secret iv in here".getBytes();
+    }
+    
+}
+```
+
+**Use this rule by referencing it:**
+``` xml
+<rule ref="category/java/security.xml/InsecureCryptoIv" />
+```
+

From c7e493f283a6b6535a14b482c057133e9d1a3046 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Mon, 23 Apr 2018 21:20:48 +0200
Subject: [PATCH 48/53] [java] Enable security ruleset for all-java.xml

---
 pmd-core/src/main/resources/rulesets/internal/all-java.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pmd-core/src/main/resources/rulesets/internal/all-java.xml b/pmd-core/src/main/resources/rulesets/internal/all-java.xml
index db34c37745..26ac051270 100644
--- a/pmd-core/src/main/resources/rulesets/internal/all-java.xml
+++ b/pmd-core/src/main/resources/rulesets/internal/all-java.xml
@@ -27,6 +27,6 @@
     <rule ref="category/java/errorprone.xml" />
     <rule ref="category/java/multithreading.xml" />
     <rule ref="category/java/performance.xml" />
-<!--     <rule ref="category/java/security.xml" /> -->
+    <rule ref="category/java/security.xml" />
 
 </ruleset>

From 6ccbd11f9dc11bc88defb557ef7c68a317a4b308 Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@microdoc.com>
Date: Tue, 24 Apr 2018 15:08:33 +0200
Subject: [PATCH 49/53] [doc] Update sample configuration of
 CyclomaticComplexity rule

---
 docs/pages/pmd/userdocs/making_rulesets.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/pages/pmd/userdocs/making_rulesets.md b/docs/pages/pmd/userdocs/making_rulesets.md
index 26ea3b1eee..1b9345a833 100644
--- a/docs/pages/pmd/userdocs/making_rulesets.md
+++ b/docs/pages/pmd/userdocs/making_rulesets.md
@@ -57,7 +57,8 @@ After you add these references it’ll look something like this:
 	<!-- Now we'll customize a rule's property value -->
 	<rule ref="category/java/design.xml/CyclomaticComplexity">
 		<properties>
-			<property name="reportLevel" value="5"/>
+			<property name="classReportLevel" value="40"/>
+			<property name="methodReportLevel" value="5"/>
 		</properties>
 	</rule>
 

From 0942ae36f835464a2f90202383d698058b8f0bdc Mon Sep 17 00:00:00 2001
From: gibarsin <gonzalo.ibars@mercadolibre.com>
Date: Tue, 24 Apr 2018 15:44:53 -0300
Subject: [PATCH 50/53] Make MultiThreadProcessor more space efficient

---
 .../pmd/processor/MultiThreadProcessor.java   | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/processor/MultiThreadProcessor.java b/pmd-core/src/main/java/net/sourceforge/pmd/processor/MultiThreadProcessor.java
index 9855299e62..f398a90db8 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/processor/MultiThreadProcessor.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/processor/MultiThreadProcessor.java
@@ -4,28 +4,26 @@
 
 package net.sourceforge.pmd.processor;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 
 import net.sourceforge.pmd.PMDConfiguration;
 import net.sourceforge.pmd.Report;
 import net.sourceforge.pmd.renderers.Renderer;
 
+
 /**
  * @author Romain Pelisse &lt;belaran@gmail.com&gt;
- *
  */
 public class MultiThreadProcessor extends AbstractPMDProcessor {
+    private final ExecutorService executor;
+    private final CompletionService<Report> completionService;
 
-    private ExecutorService executor;
-    private CompletionService<Report> completionService;
-    private List<Future<Report>> tasks = new ArrayList<>();
+    private long submittedTasks = 0L;
 
     public MultiThreadProcessor(final PMDConfiguration configuration) {
         super(configuration);
@@ -36,22 +34,21 @@ public class MultiThreadProcessor extends AbstractPMDProcessor {
 
     @Override
     protected void runAnalysis(PmdRunnable runnable) {
-        // multi-threaded execution, dispatch analysis to worker threads
-        tasks.add(completionService.submit(runnable));
+        completionService.submit(runnable);
+        submittedTasks++;
     }
 
     @Override
     protected void collectReports(List<Renderer> renderers) {
-        // Collect result analysis, waiting for termination if needed
         try {
-            for (int i = 0; i < tasks.size(); i++) {
+            for (int i = 0; i < submittedTasks; i++) {
                 final Report report = completionService.take().get();
                 super.renderReports(renderers, report);
             }
-        } catch (InterruptedException ie) {
+        } catch (final InterruptedException ie) {
             Thread.currentThread().interrupt();
-        } catch (ExecutionException ee) {
-            Throwable t = ee.getCause();
+        } catch (final ExecutionException ee) {
+            final Throwable t = ee.getCause();
             if (t instanceof RuntimeException) {
                 throw (RuntimeException) t;
             } else if (t instanceof Error) {

From 6cb0b47ef5dc6e5de175bd952cd9c8abff022c7a Mon Sep 17 00:00:00 2001
From: Andreas Dangel <andreas.dangel@adangel.org>
Date: Tue, 24 Apr 2018 22:22:48 +0200
Subject: [PATCH 51/53] [apex] Add test case, refs #776

Note: only solution right now is supressing the rule for the string.
---
 .../rule/errorprone/xml/AvoidHardcodingId.xml   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
index 8e8f7393de..d69a71476f 100644
--- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
+++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/errorprone/xml/AvoidHardcodingId.xml
@@ -74,4 +74,21 @@ public class Foo {
 }
 		]]></code>
 	</test-code>
+
+    <test-code>
+        <description>[apex] AvoidHardcodingId false positives #776</description>
+        <expected-problems>0</expected-problems>
+        <code><![CDATA[
+public class Foo {
+    void foo() {
+        // this is a false positive, we can't say, whether it's a salesforce id or not
+        @SuppressWarnings('PMD.AvoidHardcodingId')
+        String IMEI__c = '359040082913024';
+
+        // now the 6th character is non-0, definitive not a salesforce id
+        String IMEI2__c = '359041082913024';
+    }
+}
+        ]]></code>
+    </test-code>
 </test-data>

From fabc52946f817c84d38aa219035fd0e1610d3c3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Wed, 25 Apr 2018 00:19:39 -0300
Subject: [PATCH 52/53] Update changelog, refs #1048

---
 docs/pages/release_notes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 6a87a10f65..575350baca 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -145,4 +145,5 @@ Other languages are equivalent.
 *   [#1041](https://github.com/pmd/pmd/pull/1041): \[java] Make BasicProjectMemoizer thread safe. - [bergander](https://github.com/bergander)
 *   [#1042](https://github.com/pmd/pmd/pull/1042): \[java] New security rule: report usage of hard coded IV in crypto operations - [Sergey Gorbaty](https://github.com/sgorbaty)
 *   [#1044](https://github.com/pmd/pmd/pull/1044): \[java] Fix for issue #816 - [Akshat Bahety](https://github.com/akshatbahety)
+*   [#1048](https://github.com/pmd/pmd/pull/1048): \[core] Make MultiThreadProcessor more space efficient - [Gonzalo Exequiel Ibars Ingman](https://github.com/gibarsin)
 

From b8e90e44d6acb52db4d1f8aa272d7c557f919fa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Sotuyo=20Dodero?=
 <juansotuyo@gmail.com>
Date: Wed, 25 Apr 2018 01:38:12 -0300
Subject: [PATCH 53/53] Update changelog, refs #778

---
 docs/pages/release_notes.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 20013d7118..1e511ebcd7 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -14,6 +14,7 @@ This is a minor release.
 
 * [New and noteworthy](#new-and-noteworthy)
     *   [Tree transversal revision](#tree-transversal-revision)
+    *   [Swift 4.1 Support](#swift-41-support)
 * [Fixed Issues](#fixed-issues)
 * [API Changes](#api-changes)
 * [External Contributions](#external-contributions)
@@ -32,16 +33,32 @@ This change implies several false positives / unexpected results (ie: `ASTBlockS
 have been fixed; and lots of searches are now restricted to smaller search areas, which improves performance (depending on the project,
 we have measured up to 10% improvements during Type Resolution, Symbol Table analysis, and some rule's application).
 
+#### Swift 4.1 Support
+
+Thanks to major contributions from [kenji21](https://github.com/kenji21) the Swift grammar has been updated to support Swift 4.1.
+This is a major update, since the old grammar was quite dated, and we are sure all iOS developers will enjoy it.
+
+Unfortunately, this change is not compatible. The grammar elements that have been removed (ie: the keywords `__FILE__`,
+`__LINE__`, `__COLUMN__` and `__FUNCTION__`) are no longer supported. We don't usually introduce such drastic / breaking
+changes in minor releases, however, given that the whole Swift ecosystem pushes hard towards always using the latest
+versions, and that Swift needs all code and libraries to be currently compiling against the same Swift version,
+we felt strongly this change was both safe and necessary to be shipped as soon as possible. We had great feedback
+from the comunity during the processm but if you have a legitimate use case for older Swift versions, please let us know
+[on our Issue Tracke](https://github.com/pmd/pmd/issues).
+
 ### Fixed Issues
 
 *   documentation
     *   [#994](https://github.com/pmd/pmd/issues/994): \[doc] Delete duplicate page contributing.md on the website
 *   java-bestpracrtices
     *   [#370](https://github.com/pmd/pmd/issues/370): \[java] GuardLogStatementJavaUtil not considering lambdas
+*   swift
+    *   [#678](https://github.com/pmd/pmd/issues/678): \[swift][cpd] Exception when running for Swift 4 code (KeyPath)
 
 ### API Changes
 
 ### External Contributions
 
+*   [#778](https://github.com/pmd/pmd/pull/778): \[swift] Support Swift 4 grammar - [kenji21](https://github.com/kenji21)
 *   [#1002](https://github.com/pmd/pmd/pull/1002): \[doc] Delete duplicate page contributing.md on the website - [Ishan Srivastava](https://github.com/ishanSrt)
 *   [#1008](https://github.com/pmd/pmd/pull/1008): \[core] DOC: fix closing tag for &lt;pmdVersion> - [stonio](https://github.com/stonio)