Extended Objective-C grammar to accept Unicode characters in identifiers.

pmd-objectivec/etc/grammar/ObjC2.0.jj

@@ -548,9 +548,19 @@ TOKEN : { <EXPORTED_CALLBACK : "EXPORTED_CALLBACK"> }
 
 TOKEN :
 {
-   < IDENT : <IDENT_NONDIGIT> ( <IDENT_NONDIGIT> | <DIGIT>)* >
+  < IDENT : <IDENT_NONDIGIT> ( <IDENT_NONDIGIT> | <DIGIT>)* >
 | <#IDENT_NONDIGIT : <NONDIGIT> | <UNIVERSAL_CHARACTER_NAME> > // "may include other implementation-defined characters"
-| <#NONDIGIT : ["a"-"z"] | ["A"-"Z"] | "_" | "$" >
+| <#NONDIGIT : ["a"-"z"] | ["A"-"Z"] | "_" | "$" | <NONDIGIT_UNICODE> >
+| <#NONDIGIT_UNICODE : [ /* source: http://stackoverflow.com/questions/30933785 */
+   "\u0024",
+   "\u0041"-"\u005a",
+   "\u005f",
+   "\u0061"-"\u007a",
+   "\u00c0"-"\u00d6",
+   "\u00d8"-"\u00f6",
+   "\u00f8"-"\u00ff",
+   "\u0100"-"\u1fff"
+  ] >
 | <#UNIVERSAL_CHARACTER_NAME : ("\\u" <HEX_QUAD>) | ("\\U" <HEX_QUAD> <HEX_QUAD>) >
 }
 

pmd-objectivec/src/test/java/net/sourceforge/pmd/cpd/UnicodeObjectiveCTokenizerTest.java

@@ -0,0 +1,37 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package net.sourceforge.pmd.cpd;
+
+import java.io.IOException;
+
+import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+
+//Tests if the ObjectiveC tokenizer supports identifiers with unicode characters
+public class UnicodeObjectiveCTokenizerTest extends AbstractTokenizerTest {
+
+    private static final String FILENAME = "NCClient.m";
+
+    @Before
+    @Override
+    public void buildTokenizer() throws IOException {
+        this.tokenizer = new ObjectiveCTokenizer();
+        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
+    }
+
+    @Override
+    public String getSampleCode() throws IOException {
+        return IOUtils.toString(ObjectiveCTokenizer.class.getResourceAsStream(FILENAME), "UTF-8");
+    }
+
+    @Test
+    public void tokenizeTest() throws IOException {
+        this.expectedTokenCount = 10;
+        super.tokenizeTest();
+    }
+}

pmd-objectivec/src/test/resources/net/sourceforge/pmd/cpd/NCClient.m

@@ -0,0 +1,5 @@
+@import UIKit;
+
+static SecCertificateRef gNСServerLogonCertificate;
+
+@end