hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Added support for Python to CPD.

Browse Source
This commit is contained in:
Jan van Nunen
2015-01-06 17:02:02 +01:00
parent c1bc045c65
commit 43750e3799
18 changed files with 1013 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

279
pmd-python/etc/grammar/python.jj Normal file
View File

@ -0,0 +1,279 @@
/**
* This Python 2.7 grammar was copied from the PyDev Project. (http://www.pydev.org/)
*
* Original source file:
* https://github.com/aptana/Pydev/blob/development/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar27/python.jjt (commit 32950d534139f286e03d34795aec99edab09c04c)
*/
options {
BUILD_PARSER=false;
CACHE_TOKENS=true;
STATIC=false;
UNICODE_INPUT = true;
USER_CHAR_STREAM=true;
}
PARSER_BEGIN(PythonParser)
package net.sourceforge.pmd.lang.python.ast;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PythonParser {
}
PARSER_END(PythonParser)
SKIP :
{
<SPACE: " ">
| "\t"
| "\014"
| <CONTINUATION: ("\\") ("\r\n"|"\n"|"\r")>
| <NEWLINE: ("\r\n"|"\n"|"\r")>
| <TRAILING_COMMENT: "#" (~["\n","\r"])* >
}
TOKEN : /* SEPARATORS */
{
< LPAREN: "(" >
| < RPAREN: ")" >
| < LBRACE: "{" >
| < RBRACE: "}" >
| < LBRACKET: "[" >
| < RBRACKET: "]" >
| < SEMICOLON: ";" >
| < COMMA: "," >
| < DOT: "." >
| < COLON: ":" >
}
TOKEN : /* OPERATORS */
{
< PLUS: "+" >
| < MINUS: "-" >
| < MULTIPLY: "*" >
| < DIVIDE: "/" >
| < FLOORDIVIDE: "//" >
| < POWER: "**" >
| < LSHIFT: "<<" >
| < RSHIFT: ">>" >
| < MODULO: "%" >
| < NOT: "~" >
| < XOR: "^" >
| < OR: "|" >
| < AND: "&" >
| < EQUAL: "=" >
| < GREATER: ">" >
| < LESS: "<" >
| < EQEQUAL: "==" >
| < EQLESS: "<=" >
| < EQGREATER: ">=" >
| < LESSGREATER: "<>" >
| < NOTEQUAL: "!=" >
| < PLUSEQ: "+=" >
| < MINUSEQ: "-=" >
| < MULTIPLYEQ: "*=" >
| < DIVIDEEQ: "/=" >
| < FLOORDIVIDEEQ: "//=" >
| < MODULOEQ: "%=" >
| < ANDEQ: "&=" >
| < OREQ: "|=" >
| < XOREQ: "^=" >
| < LSHIFTEQ: "<<=" >
| < RSHIFTEQ: ">>=" >
| < POWEREQ: "**=" >
}
TOKEN : /* KEYWORDS */
{
< OR_BOOL: "or" >
| < AND_BOOL: "and" >
| < NOT_BOOL: "not" >
| < IS: "is" >
| < IN: "in" >
| < LAMBDA: "lambda" >
| < IF: "if" >
| < ELSE: "else" >
| < ELIF: "elif" >
| < WHILE: "while" >
| < FOR: "for" >
| < TRY: "try" >
| < EXCEPT: "except" >
| < DEF: "def" >
| < CLASS: "class" >
| < FINALLY: "finally" >
| < PRINT: "print" >
| < PASS: "pass" >
| < BREAK: "break" >
| < CONTINUE: "continue" >
| < RETURN: "return" >
| < YIELD: "yield" >
| < IMPORT: "import" >
| < FROM: "from" >
| < DEL: "del" >
| < RAISE: "raise" >
| < GLOBAL: "global" >
| < EXEC: "exec" >
| < ASSERT: "assert" >
| < AS: "as" >
| <WITH: "with" >
| < AT: "@" >
}
TOKEN : /* Python identifiers */
{
< NAME: <LETTER> ( <LETTER> | <DIGIT>)* >
| < #LETTER: ["_","a"-"z","A"-"Z"] >
}
TOKEN : /* Numeric literals */
{
< DECNUMBER:
["1"-"9"] (["0"-"9"])* (["l", "L"])?
| "0"
>
| < HEXNUMBER: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+ (["l","L"])? >
| < OCTNUMBER: "0" (["o","O"])? (["0"-"7"])* (["l","L"])? >
| < BINNUMBER: "0" (["b","B"])? (["0"-"1"])* (["l","L"])? >
|
< FLOAT:
(["0"-"9"])+ "." (["0"-"9"])* (<EXPONENT>)?
| "." (["0"-"9"])+ (<EXPONENT>)?
| (["0"-"9"])+ <EXPONENT>
>
| < COMPLEX: (<DECNUMBER> | <FLOAT> | "0" <DECNUMBER> ) ["j", "J"]>
| < #EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+ >
| < #DIGIT: ["0" - "9"] >
}
MORE : /* Strings */
{
< (["u", "U"]) (["r", "R"])? "'" > : IN_USTRING11
| < (["u", "U"]) (["r", "R"])? "\"" > : IN_USTRING21
| < (["u", "U"]) (["r", "R"])? "'''" > : IN_USTRING13
| < (["u", "U"]) (["r", "R"])? "\"\"\"" > : IN_USTRING23
| < (["b", "B"]) (["r", "R"])? "'" > : IN_BSTRING11
| < (["b", "B"]) (["r", "R"])? "\"" > : IN_BSTRING21
| < (["b", "B"]) (["r", "R"])? "'''" > : IN_BSTRING13
| < (["b", "B"]) (["r", "R"])? "\"\"\"" > : IN_BSTRING23
| < (["r", "R"])? "'" > : IN_STRING11
| < (["r", "R"])? "\"" > : IN_STRING21
| < (["r", "R"])? "'''" > : IN_STRING13
| < (["r", "R"])? "\"\"\"" > : IN_STRING23
}
<IN_STRING11> TOKEN : { <SINGLE_STRING: "'"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING21> TOKEN : { <SINGLE_STRING2: "\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING13> TOKEN : { <TRIPLE_STRING: "'''"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING23> TOKEN : { <TRIPLE_STRING2: "\"\"\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_BSTRING11> TOKEN : { <SINGLE_BSTRING: "'"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_BSTRING21> TOKEN : { <SINGLE_BSTRING2: "\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_BSTRING13> TOKEN : { <TRIPLE_BSTRING: "'''"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_BSTRING23> TOKEN : { <TRIPLE_BSTRING2: "\"\"\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING11> TOKEN : { <SINGLE_USTRING: "'"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING21> TOKEN : { <SINGLE_USTRING2: "\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING13> TOKEN : { <TRIPLE_USTRING: "'''"> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_USTRING23> TOKEN : { <TRIPLE_USTRING2: "\"\"\""> {
matchedToken.image = image.toString(); } : DEFAULT}
<IN_STRING11> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_STRING1NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_STRING1NLC
}
<IN_STRING21> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_STRING2NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_STRING2NLC
}
<IN_USTRING11> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_USTRING1NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_USTRING1NLC
}
<IN_USTRING21> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_USTRING2NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_USTRING2NLC
}
<IN_BSTRING11> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_BSTRING1NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_BSTRING1NLC
}
<IN_BSTRING21> MORE:
{
<"\\\r\n"> { image.setLength(image.length()-3); } : IN_BSTRING2NLC
| <("\\" ("\n"|"\r"))> { image.setLength(image.length()-2); } : IN_BSTRING2NLC
}
<IN_STRING1NLC> MORE:
{
<""> : IN_STRING11
}
<IN_STRING2NLC> MORE:
{
<""> : IN_STRING21
}
<IN_USTRING1NLC> MORE:
{
<""> : IN_USTRING11
}
<IN_USTRING2NLC> MORE:
{
<""> : IN_USTRING21
}
<IN_BSTRING1NLC> MORE:
{
<""> : IN_BSTRING11
}
<IN_BSTRING2NLC> MORE:
{
<""> : IN_BSTRING21
}
<IN_STRING11, IN_USTRING11, IN_BSTRING11> MORE: { <("\\" ("\\"|"'")) | ~["\n","\r"]> }
<IN_STRING21, IN_USTRING21, IN_BSTRING21> MORE: { <("\\" ("\\"|"\"")) | ~["\n","\r"]> }
<IN_STRING13, IN_STRING23, IN_USTRING13, IN_USTRING23, IN_BSTRING13, IN_BSTRING23> MORE:
{
<"\r\n"> {
int l = image.length();
image.setLength(l-1);
image.setCharAt(l-2, '\n');
}
| <"\n">
| <"\r"> { image.setCharAt(image.length()-1, '\n'); }
| <~["\n","\r"]>
| <"\\" ~["\n","\r"]>
}

96
pmd-python/pom.xml Normal file
View File

@ -0,0 +1,96 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>pmd-python</artifactId>
<name>PMD Python</name>
<parent>
<groupId>net.sourceforge.pmd</groupId>
<artifactId>pmd</artifactId>
<version>5.2.4-SNAPSHOT</version>
</parent>
<properties>
<config.basedir>${basedir}/../pmd-core</config.basedir>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<useDefaultDelimiters>false</useDefaultDelimiters>
<delimiters>
<delimiter>${*}</delimiter>
</delimiters>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<inherited>true</inherited>
<executions>
<execution>
<id>generate-sources</id>
<phase>generate-sources</phase>
<configuration>
<target>
<ant antfile="src/main/ant/alljavacc.xml">
<property name="target" value="${project.build.directory}/generated-sources/javacc" />
<property name="javacc.jar" value="${settings.localRepository}/net/java/dev/javacc/javacc/${javacc.version}/javacc-${javacc.version}.jar" />
</ant>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-javacc-generated-sources</id>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.build.directory}/generated-sources/javacc</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<configuration>
<xdocDirectory>${project.build.directory}/generated-xdocs</xdocDirectory>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>net.sourceforge.pmd</groupId>
<artifactId>pmd-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.sourceforge.pmd</groupId>
<artifactId>pmd-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

45
pmd-python/src/main/ant/alljavacc.xml Normal file
View File

@ -0,0 +1,45 @@
<project name="pmd" default="alljavacc" basedir="../../">
<property name="javacc-home.path" value="target/lib" />
<target name="alljavacc"
description="Generates all JavaCC aspects within PMD"
depends="checkUpToDate,init,pythonjavacc,cleanup" />
<target name="checkUpToDate">
<uptodate property="javaccBuildNotRequired" targetfile="${target}/last-generated-timestamp">
<srcfiles dir="etc/grammar" includes="*.jj*"/>
</uptodate>
<echo message="up to date check: javaccBuildNotRequired=${javaccBuildNotRequired}"/>
</target>
<target name="init" unless="javaccBuildNotRequired">
<mkdir dir="${javacc-home.path}" />
<copy file="${javacc.jar}" tofile="${javacc-home.path}/javacc.jar" />
<mkdir dir="${target}"/>
<touch file="${target}/last-generated-timestamp"/>
</target>
<target name="cleanup">
<delete dir="${javacc-home.path}" />
</target>
<target name="pythonjavacc" description="Generates the Python grammar" unless="javaccBuildNotRequired">
<delete dir="${target}/net/sourceforge/pmd/lang/python/ast" />
<mkdir dir="${target}/net/sourceforge/pmd/lang/python/ast" />
<!-- Ensure generated using CharStream interface -->
<javacc static="false"
usercharstream="true"
target="etc/grammar/python.jj"
outputdirectory="${target}/net/sourceforge/pmd/lang/python/ast"
javacchome="${javacc-home.path}" />
<replace file="${target}/net/sourceforge/pmd/lang/python/ast/PythonParserTokenManager.java"
token="class PythonParserTokenManager"
value="class PythonParserTokenManager extends net.sourceforge.pmd.lang.ast.AbstractTokenManager" />
<delete file="${target}/net/sourceforge/pmd/lang/python/ast/CharStream.java" />
<delete file="${target}/net/sourceforge/pmd/lang/python/ast/ParseException.java" />
<delete file="${target}/net/sourceforge/pmd/lang/python/ast/TokenMgrError.java" />
</target>
</project>

18
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonLanguage.java Normal file
View File

@ -0,0 +1,18 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
/**
* Defines the Language module for Python
*/
public class PythonLanguage extends AbstractLanguage {
/**
* Creates a new instance of {@link PythonLanguage} with the default extensions for python files.
*/
public PythonLanguage() {
super("Python", "python", new PythonTokenizer(), ".py");
}
}

46
pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java Normal file
View File

@ -0,0 +1,46 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import net.sourceforge.pmd.lang.python.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
import org.apache.commons.io.IOUtils;
public class PythonTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
Reader reader = null;
try {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
reader = new StringReader(buffer.toString());
reader = IOUtil.skipBOM(reader);
TokenManager tokenManager = languageVersionHandler.getParser(
languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
Token currentToken = (Token) tokenManager.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenManager.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode);
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} finally {
IOUtils.closeQuietly(reader);
}
}
}

25
pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonHandler.java Normal file
View File

@ -0,0 +1,25 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.python;
import net.sourceforge.pmd.lang.AbstractLanguageVersionHandler;
import net.sourceforge.pmd.lang.Parser;
import net.sourceforge.pmd.lang.ParserOptions;
import net.sourceforge.pmd.lang.rule.RuleViolationFactory;
/**
* Implementation of LanguageVersionHandler for the Python Language.
*/
public class PythonHandler extends AbstractLanguageVersionHandler {
@Override
public RuleViolationFactory getRuleViolationFactory() {
throw new UnsupportedOperationException("getRuleViolationFactory() is not supported for Python");
}
@Override
public Parser getParser(ParserOptions parserOptions) {
return new PythonParser(parserOptions);
}
}

25
pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java Normal file
View File

@ -0,0 +1,25 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.python;
import net.sourceforge.pmd.lang.BaseLanguageModule;
/**
* Implementation of the Python Language Module.
*/
public class PythonLanguageModule extends BaseLanguageModule {
/** The name, that can be used to display the language in UI. */
public static final String NAME = "Python";
/** The internal name. */
public static final String TERSE_NAME = "python";
/**
* Creates a new instance of {@link PythonLanguageModule} with the default file extensions for Python.
*/
public PythonLanguageModule() {
super(NAME, null, TERSE_NAME, null, "py");
addVersion("", new PythonHandler(), true);
}
}

49
pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonParser.java Normal file
View File

@ -0,0 +1,49 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.python;
import java.io.Reader;
import java.util.Map;
import net.sourceforge.pmd.lang.AbstractParser;
import net.sourceforge.pmd.lang.ParserOptions;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.AbstractTokenManager;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.ParseException;
/**
* Adapter for the Python Parser.
*/
public class PythonParser extends AbstractParser {
/**
* Creates a new Python Parser.
* @param parserOptions the options
*/
public PythonParser(ParserOptions parserOptions) {
super(parserOptions);
}
@Override
public TokenManager createTokenManager(Reader source) {
return new PythonTokenManager(source);
}
@Override
public boolean canParse() {
return false;
}
@Override
public Node parse(String fileName, Reader source) throws ParseException {
AbstractTokenManager.setFileName(fileName);
throw new UnsupportedOperationException("parse(Reader) is not supported for Python");
}
@Override
public Map<Integer, String> getSuppressMap() {
throw new UnsupportedOperationException("getSuppressMap() is not supported for Python");
}
}

34
pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java Normal file
View File

@ -0,0 +1,34 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.python;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.SimpleCharStream;
import net.sourceforge.pmd.lang.python.ast.PythonParserTokenManager;
/**
* Python Token Manager implementation.
*/
public class PythonTokenManager implements TokenManager {
private final PythonParserTokenManager tokenManager;
/**
* Creates a new Python Token Manager from the given source code.
* @param source the source code
*/
public PythonTokenManager(Reader source) {
tokenManager = new PythonParserTokenManager(new SimpleCharStream(source));
}
public Object getNextToken() {
return tokenManager.getNextToken();
}
@Override
public void setFileName(String fileName) {
PythonParserTokenManager.setFileName(fileName);
}
}

1
pmd-python/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language Normal file
View File

@ -0,0 +1 @@
net.sourceforge.pmd.cpd.PythonLanguage

1
pmd-python/src/main/resources/META-INF/services/net.sourceforge.pmd.lang.Language Normal file
View File

@ -0,0 +1 @@
net.sourceforge.pmd.lang.python.PythonLanguageModule

3
pmd-python/src/site/markdown/index.md Normal file
View File

@ -0,0 +1,3 @@
# PMD Python
Only CPD is supported. There are no PMD rules for Python.

12
pmd-python/src/site/site.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project
xmlns="http://maven.apache.org/DECORATION/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/DECORATION/1.1.0 http://maven.apache.org/xsd/decoration-1.1.0.xsd"
name="PMD Python">
<body>
<menu ref="parent"/>
<menu ref="reports"/>
</body>
</project>

32
pmd-python/src/test/java/net/sourceforge/pmd/LanguageVersionDiscovererTest.java Normal file
View File

@ -0,0 +1,32 @@
package net.sourceforge.pmd;
import static org.junit.Assert.assertEquals;
import java.io.File;
import junit.framework.JUnit4TestAdapter;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersion;
import net.sourceforge.pmd.lang.LanguageVersionDiscoverer;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import org.junit.Test;
public class LanguageVersionDiscovererTest {
/**
* Test on Python file with default version
*/
@Test
public void testPlsql() {
LanguageVersionDiscoverer discoverer = new LanguageVersionDiscoverer();
File plsqlFile = new File("/path/to/MY_PACKAGE.py");
LanguageVersion languageVersion = discoverer.getDefaultLanguageVersionForFile(plsqlFile);
assertEquals("LanguageVersion must be Python!", LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion(), languageVersion);
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(LanguageVersionDiscovererTest.class);
}
}

27
pmd-python/src/test/java/net/sourceforge/pmd/LanguageVersionTest.java Normal file
View File

@ -0,0 +1,27 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd;
import java.util.Arrays;
import java.util.Collection;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersion;
import net.sourceforge.pmd.lang.python.PythonLanguageModule;
import org.junit.runners.Parameterized.Parameters;
public class LanguageVersionTest extends AbstractLanguageVersionTest {
public LanguageVersionTest(String name, String terseName, String version, LanguageVersion expected) {
super(name, terseName, version, expected);
}
@Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{ PythonLanguageModule.NAME, PythonLanguageModule.TERSE_NAME, "", LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion() }
});
}
}

40
pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java Normal file
View File

@ -0,0 +1,40 @@
/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
import net.sourceforge.pmd.testframework.StreamUtil;
import org.junit.Before;
import org.junit.Test;
public class PythonTokenizerTest extends AbstractTokenizerTest {
private static final String FILENAME = "sample-python.py";
@Before
@Override
public void buildTokenizer() {
this.tokenizer = new PythonTokenizer();
this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
}
@Override
public String getSampleCode() {
return StreamUtil.toString(PythonTokenizer.class.getResourceAsStream(FILENAME));
}
@Test
public void tokenizeTest() throws IOException {
this.expectedTokenCount = 1218;
super.tokenizeTest();
}
public static junit.framework.Test suite() {
return new junit.framework.JUnit4TestAdapter(PythonTokenizerTest.class);
}
}

279
pmd-python/src/test/resources/net/sourceforge/pmd/cpd/sample-python.py Normal file
View File

@ -0,0 +1,279 @@
# Example source code copied from the Django project on GitHub
# https://github.com/django/django/blob/master/django/core/handlers/base.py
from __future__ import unicode_literals
import logging
import sys
import types
from django import http
from django.conf import settings
from django.core import urlresolvers
from django.core import signals
from django.core.exceptions import MiddlewareNotUsed, PermissionDenied, SuspiciousOperation
from django.db import connections, transaction
from django.http.multipartparser import MultiPartParserError
from django.utils.encoding import force_text
from django.utils.module_loading import import_string
from django.utils import six
from django.views import debug
logger = logging.getLogger('django.request')
class BaseHandler(object):
# Changes that are always applied to a response (in this order).
response_fixes = [
http.fix_location_header,
http.conditional_content_removal,
]
def __init__(self):
self._request_middleware = None
self._view_middleware = None
self._template_response_middleware = None
self._response_middleware = None
self._exception_middleware = None
def load_middleware(self):
"""
Populate middleware lists from settings.MIDDLEWARE_CLASSES.
Must be called after the environment is fixed (see __call__ in subclasses).
"""
self._view_middleware = []
self._template_response_middleware = []
self._response_middleware = []
self._exception_middleware = []
request_middleware = []
for middleware_path in settings.MIDDLEWARE_CLASSES:
mw_class = import_string(middleware_path)
try:
mw_instance = mw_class()
except MiddlewareNotUsed as exc:
if settings.DEBUG:
if six.text_type(exc):
logger.debug('MiddlewareNotUsed(%r): %s', middleware_path, exc)
else:
logger.debug('MiddlewareNotUsed: %r', middleware_path)
continue
if hasattr(mw_instance, 'process_request'):
request_middleware.append(mw_instance.process_request)
if hasattr(mw_instance, 'process_view'):
self._view_middleware.append(mw_instance.process_view)
if hasattr(mw_instance, 'process_template_response'):
self._template_response_middleware.insert(0, mw_instance.process_template_response)
if hasattr(mw_instance, 'process_response'):
self._response_middleware.insert(0, mw_instance.process_response)
if hasattr(mw_instance, 'process_exception'):
self._exception_middleware.insert(0, mw_instance.process_exception)
# We only assign to this when initialization is complete as it is used
# as a flag for initialization being complete.
self._request_middleware = request_middleware
def make_view_atomic(self, view):
non_atomic_requests = getattr(view, '_non_atomic_requests', set())
for db in connections.all():
if (db.settings_dict['ATOMIC_REQUESTS']
and db.alias not in non_atomic_requests):
view = transaction.atomic(using=db.alias)(view)
return view
def get_exception_response(self, request, resolver, status_code):
try:
callback, param_dict = resolver.resolve_error_handler(status_code)
response = callback(request, **param_dict)
except:
signals.got_request_exception.send(sender=self.__class__, request=request)
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
return response
def get_response(self, request):
"Returns an HttpResponse object for the given HttpRequest"
# Setup default url resolver for this thread, this code is outside
# the try/except so we don't get a spurious "unbound local
# variable" exception in the event an exception is raised before
# resolver is set
urlconf = settings.ROOT_URLCONF
urlresolvers.set_urlconf(urlconf)
resolver = urlresolvers.RegexURLResolver(r'^/', urlconf)
try:
response = None
# Apply request middleware
for middleware_method in self._request_middleware:
response = middleware_method(request)
if response:
break
if response is None:
if hasattr(request, 'urlconf'):
# Reset url resolver with a custom urlconf.
urlconf = request.urlconf
urlresolvers.set_urlconf(urlconf)
resolver = urlresolvers.RegexURLResolver(r'^/', urlconf)
resolver_match = resolver.resolve(request.path_info)
callback, callback_args, callback_kwargs = resolver_match
request.resolver_match = resolver_match
# Apply view middleware
for middleware_method in self._view_middleware:
response = middleware_method(request, callback, callback_args, callback_kwargs)
if response:
break
if response is None:
wrapped_callback = self.make_view_atomic(callback)
try:
response = wrapped_callback(request, *callback_args, **callback_kwargs)
except Exception as e:
# If the view raised an exception, run it through exception
# middleware, and if the exception middleware returns a
# response, use that. Otherwise, reraise the exception.
for middleware_method in self._exception_middleware:
response = middleware_method(request, e)
if response:
break
if response is None:
raise
# Complain if the view returned None (a common error).
if response is None:
if isinstance(callback, types.FunctionType): # FBV
view_name = callback.__name__
else: # CBV
view_name = callback.__class__.__name__ + '.__call__'
raise ValueError("The view %s.%s didn't return an HttpResponse object. It returned None instead."
% (callback.__module__, view_name))
# If the response supports deferred rendering, apply template
# response middleware and then render the response
if hasattr(response, 'render') and callable(response.render):
for middleware_method in self._template_response_middleware:
response = middleware_method(request, response)
# Complain if the template response middleware returned None (a common error).
if response is None:
raise ValueError(
"%s.process_template_response didn't return an "
"HttpResponse object. It returned None instead."
% (middleware_method.__self__.__class__.__name__))
response = response.render()
except http.Http404 as e:
logger.warning('Not Found: %s', request.path,
extra={
'status_code': 404,
'request': request
})
if settings.DEBUG:
response = debug.technical_404_response(request, e)
else:
response = self.get_exception_response(request, resolver, 404)
except PermissionDenied:
logger.warning(
'Forbidden (Permission denied): %s', request.path,
extra={
'status_code': 403,
'request': request
})
response = self.get_exception_response(request, resolver, 403)
except MultiPartParserError:
logger.warning(
'Bad request (Unable to parse request body): %s', request.path,
extra={
'status_code': 400,
'request': request
})
response = self.get_exception_response(request, resolver, 400)
except SuspiciousOperation as e:
# The request logger receives events for any problematic request
# The security logger receives events for all SuspiciousOperations
security_logger = logging.getLogger('django.security.%s' %
e.__class__.__name__)
security_logger.error(
force_text(e),
extra={
'status_code': 400,
'request': request
})
if settings.DEBUG:
return debug.technical_500_response(request, *sys.exc_info(), status_code=400)
response = self.get_exception_response(request, resolver, 400)
except SystemExit:
# Allow sys.exit() to actually exit. See tickets #1023 and #4701
raise
except: # Handle everything else.
# Get the exception info now, in case another exception is thrown later.
signals.got_request_exception.send(sender=self.__class__, request=request)
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
try:
# Apply response middleware, regardless of the response
for middleware_method in self._response_middleware:
response = middleware_method(request, response)
# Complain if the response middleware returned None (a common error).
if response is None:
raise ValueError(
"%s.process_response didn't return an "
"HttpResponse object. It returned None instead."
% (middleware_method.__self__.__class__.__name__))
response = self.apply_response_fixes(request, response)
except: # Any exception should be gathered and handled
signals.got_request_exception.send(sender=self.__class__, request=request)
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
response._closable_objects.append(request)
return response
def handle_uncaught_exception(self, request, resolver, exc_info):
"""
Processing for any otherwise uncaught exceptions (those that will
generate HTTP 500 responses). Can be overridden by subclasses who want
customised 500 handling.
Be *very* careful when overriding this because the error could be
caused by anything, so assuming something like the database is always
available would be an error.
"""
if settings.DEBUG_PROPAGATE_EXCEPTIONS:
raise
logger.error('Internal Server Error: %s', request.path,
exc_info=exc_info,
extra={
'status_code': 500,
'request': request
}
)
if settings.DEBUG:
return debug.technical_500_response(request, *exc_info)
# If Http500 handler is not installed, re-raise last exception
if resolver.urlconf_module is None:
six.reraise(*exc_info)
# Return an HttpResponse that displays a friendly error message.
callback, param_dict = resolver.resolve_error_handler(500)
return callback(request, **param_dict)
def apply_response_fixes(self, request, response):
"""
Applies each of the functions in self.response_fixes to the request and
response, modifying the response in the process. Returns the new
response.
"""
for func in self.response_fixes:
response = func(request, response)
return response

1
pom.xml
View File

@ -866,6 +866,7 @@
<module>pmd-jsp</module>
<module>pmd-php</module>
<module>pmd-plsql</module>
<module>pmd-python</module>
<module>pmd-ruby</module>
<module>pmd-test</module>
<module>pmd-vm</module>