pmd/pmd-python/etc/grammar/Python.jj

/**
 * This Python 2.7 grammar was copied from the PyDev Project. (http://www.pydev.org/)
 *
 * Original source file:
 * https://github.com/aptana/Pydev/blob/development/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar27/python.jjt (commit 32950d534139f286e03d34795aec99edab09c04c)
 */

options {
  BUILD_PARSER=false;
  CACHE_TOKENS=true;
  STATIC=false;
  UNICODE_INPUT = true;
  USER_CHAR_STREAM=true;
}

PARSER_BEGIN(PythonParserImpl)

package net.sourceforge.pmd.lang.python.ast;

import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;

public class PythonParserImpl {

}

PARSER_END(PythonParserImpl)

SKIP :
{
    <SPACE: " ">
|   "\t"
|   "\014"
|   <CONTINUATION: ("\\") ("\r\n"|"\n"|"\r")>
|   <NEWLINE: ("\r\n"|"\n"|"\r")>
}

SPECIAL_TOKEN :
{
    <TRAILING_COMMENT: "#" (~["\n","\r"])* >
}

TOKEN : /* SEPARATORS */
{
    < LPAREN: "(" >
|   < RPAREN: ")" >
|   < LBRACE: "{" >
|   < RBRACE: "}" >
|   < LBRACKET: "[" >
|   < RBRACKET: "]" >
|   < SEMICOLON: ";" >
|   < COMMA: "," >
|   < DOT: "." >
|   < COLON: ":" >
|   < BACKTICK: "`" >
}


TOKEN : /* OPERATORS */
{
    < PLUS: "+" >
|   < MINUS: "-" >
|   < MULTIPLY: "*" >
|   < DIVIDE: "/" >
|   < FLOORDIVIDE: "//" >
|   < POWER: "**" >
|   < LSHIFT: "<<" >
|   < RSHIFT: ">>" >
|   < MODULO: "%" >
|   < NOT: "~" >
|   < XOR: "^" >
|   < OR: "|" >
|   < AND: "&" >
|   < EQUAL: "=" >
|   < GREATER: ">" >
|   < LESS: "<" >
|   < EQEQUAL: "==" >
|   < EQLESS: "<=" >
|   < EQGREATER: ">=" >
|   < LESSGREATER: "<>" >
|   < NOTEQUAL: "!=" >
|   < PLUSEQ: "+=" >
|   < MINUSEQ: "-=" >
|   < MULTIPLYEQ: "*=" >
|   < DIVIDEEQ: "/=" >
|   < FLOORDIVIDEEQ: "//=" >
|   < MODULOEQ: "%=" >
|   < ANDEQ: "&=" >
|   < OREQ: "|=" >
|   < XOREQ: "^=" >
|   < LSHIFTEQ: "<<=" >
|   < RSHIFTEQ: ">>=" >
|   < POWEREQ: "**=" >
}

TOKEN : /* KEYWORDS */
{
    < OR_BOOL: "or" >
|   < AND_BOOL: "and" >
|   < NOT_BOOL: "not" >
|   < IS: "is" >
|   < IN: "in" >
|   < LAMBDA: "lambda" >
|   < IF: "if" >
|   < ELSE: "else" >
|   < ELIF: "elif" >
|   < WHILE: "while" >
|   < FOR: "for" >
|   < TRY: "try" >
|   < EXCEPT: "except" >
|   < DEF: "def" >
|   < CLASS: "class" >
|   < FINALLY: "finally" >
|   < PRINT: "print" >
|   < PASS: "pass" >
|   < BREAK: "break" >
|   < CONTINUE: "continue" >
|   < RETURN: "return" >
|   < YIELD: "yield" >
|   < IMPORT: "import" >
|   < FROM: "from" >
|   < DEL: "del" >
|   < RAISE: "raise" >
|   < GLOBAL: "global" >
|   < EXEC: "exec" >
|   < ASSERT: "assert" >
|   < AS: "as" >
|   <WITH: "with" >
|   < AT: "@" >
}


TOKEN : /* Python identifiers */
{
    < NAME: <LETTER> ( <LETTER> | <DIGIT>)* >
|   < #LETTER: ["_","a"-"z","A"-"Z"] >
}


TOKEN : /* Numeric literals */
{
    < DECNUMBER:
        ["1"-"9"] (["0"-"9"])* (["l", "L"])?
      | "0"
      >
|   < HEXNUMBER: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+ (["l","L"])? >
|   < OCTNUMBER: "0" (["o","O"])? (["0"-"7"])* (["l","L"])? >
|   < BINNUMBER: "0" (["b","B"])? (["0"-"1"])* (["l","L"])? >
|
    < FLOAT:
        (["0"-"9"])+ "." (["0"-"9"])* (<EXPONENT>)?
      | "." (["0"-"9"])+ (<EXPONENT>)?
      | (["0"-"9"])+ <EXPONENT>
      >
|   < COMPLEX: (<DECNUMBER> | <FLOAT> | "0" <DECNUMBER> ) ["j", "J"]>
|   < #EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+ >
|   < #DIGIT: ["0" - "9"] >
}

MORE : /* Strings */
{
    < (["u", "U"]) (["r", "R"])? "'" > :  IN_USTRING11
|   < (["u", "U"]) (["r", "R"])? "\"" > :  IN_USTRING21
|   < (["u", "U"]) (["r", "R"])? "'''" > :  IN_USTRING13
|   < (["u", "U"]) (["r", "R"])? "\"\"\"" > :  IN_USTRING23

|   < (["b", "B"]) (["r", "R"])? "'" > :  IN_BSTRING11
|   < (["b", "B"]) (["r", "R"])? "\"" > :  IN_BSTRING21
|   < (["b", "B"]) (["r", "R"])? "'''" > :  IN_BSTRING13
|   < (["b", "B"]) (["r", "R"])? "\"\"\"" > :  IN_BSTRING23

|   < (["r", "R"])? "'" > :  IN_STRING11
|   < (["r", "R"])? "\"" > :  IN_STRING21
|   < (["r", "R"])? "'''" > :  IN_STRING13
|   < (["r", "R"])? "\"\"\"" > :  IN_STRING23
}

<IN_STRING11> TOKEN : { <SINGLE_STRING: "'">        : DEFAULT}
<IN_STRING21> TOKEN : { <SINGLE_STRING2: "\"">      : DEFAULT}
<IN_STRING13> TOKEN : { <TRIPLE_STRING: "'''">      : DEFAULT}
<IN_STRING23> TOKEN : { <TRIPLE_STRING2: "\"\"\"">  : DEFAULT}

<IN_BSTRING11> TOKEN : { <SINGLE_BSTRING: "'">      : DEFAULT}
<IN_BSTRING21> TOKEN : { <SINGLE_BSTRING2: "\"">    : DEFAULT}
<IN_BSTRING13> TOKEN : { <TRIPLE_BSTRING: "'''">    : DEFAULT}
<IN_BSTRING23> TOKEN : { <TRIPLE_BSTRING2: "\"\"\"">: DEFAULT}

<IN_USTRING11> TOKEN : { <SINGLE_USTRING: "'">      : DEFAULT}
<IN_USTRING21> TOKEN : { <SINGLE_USTRING2: "\"">    : DEFAULT}
<IN_USTRING13> TOKEN : { <TRIPLE_USTRING: "'''">    : DEFAULT}
<IN_USTRING23> TOKEN : { <TRIPLE_USTRING2: "\"\"\"">: DEFAULT}


<IN_STRING11, IN_USTRING11, IN_BSTRING11> MORE: {
     "\\'"
}
<IN_STRING21, IN_USTRING21, IN_BSTRING21> MORE: {
    "\\\""
}

<IN_STRING11, IN_STRING21,
 IN_USTRING11, IN_USTRING21,
 IN_BSTRING11, IN_BSTRING21> MORE:
{
    // escaping a newline ignores it, this is handled by the token document
    "\\\r\n" | "\\\n" | "\\\r"
    | "\\\\"
    | < ~["\n","\r"] >
}

// NLs are normalized in triple-quoted strings
<IN_STRING13, IN_STRING23, IN_USTRING13, IN_USTRING23, IN_BSTRING13, IN_BSTRING23> MORE:
{
    "\r\n" {
        int l = image.length();
        image.setLength(l-1);
        image.setCharAt(l-2, '\n');
    }
|   "\n"
|   "\r" { image.setCharAt(image.length()-1, '\n'); }
|   < ~["\n","\r"] >
|   < "\\" ~["\n","\r"] >
}