Introduce end-of-line normalization

References pmd/build-tools#2

pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonLanguage.java

@@ -1,19 +1,19 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.cpd;
-
-/**
- * Defines the Language module for Python
- */
-public class PythonLanguage extends AbstractLanguage {
-
-    /**
-     * Creates a new instance of {@link PythonLanguage} with the default
-     * extensions for python files.
-     */
-    public PythonLanguage() {
-        super("Python", "python", new PythonTokenizer(), ".py");
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+/**
+ * Defines the Language module for Python
+ */
+public class PythonLanguage extends AbstractLanguage {
+
+    /**
+     * Creates a new instance of {@link PythonLanguage} with the default
+     * extensions for python files.
+     */
+    public PythonLanguage() {
+        super("Python", "python", new PythonTokenizer(), ".py");
+    }
+}

pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java

@@ -1,52 +1,52 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.cpd;
-
-import java.io.Reader;
-import java.io.StringReader;
-
-import org.apache.commons.io.IOUtils;
-
-import net.sourceforge.pmd.lang.LanguageRegistry;
-import net.sourceforge.pmd.lang.LanguageVersionHandler;
-import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.TokenMgrError;
-import net.sourceforge.pmd.lang.python.PythonLanguageModule;
-import net.sourceforge.pmd.lang.python.ast.Token;
-import net.sourceforge.pmd.util.IOUtil;
-
-/**
- * The Python tokenizer.
- */
-public class PythonTokenizer implements Tokenizer {
-
-    @Override
-    public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
-        StringBuilder buffer = sourceCode.getCodeBuffer();
-        Reader reader = null;
-        try {
-            LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
-                    .getDefaultVersion().getLanguageVersionHandler();
-            reader = new StringReader(buffer.toString());
-            reader = IOUtil.skipBOM(reader);
-            TokenManager tokenManager = languageVersionHandler
-                    .getParser(languageVersionHandler.getDefaultParserOptions())
-                    .getTokenManager(sourceCode.getFileName(), reader);
-            Token currentToken = (Token) tokenManager.getNextToken();
-            while (currentToken.image.length() > 0) {
-                tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
-                currentToken = (Token) tokenManager.getNextToken();
-            }
-            tokenEntries.add(TokenEntry.getEOF());
-            System.err.println("Added " + sourceCode);
-        } catch (TokenMgrError err) {
-            err.printStackTrace();
-            System.err.println("Skipping " + sourceCode + " due to parse error");
-            tokenEntries.add(TokenEntry.getEOF());
-        } finally {
-            IOUtils.closeQuietly(reader);
-        }
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.commons.io.IOUtils;
+
+import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.LanguageVersionHandler;
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.TokenMgrError;
+import net.sourceforge.pmd.lang.python.PythonLanguageModule;
+import net.sourceforge.pmd.lang.python.ast.Token;
+import net.sourceforge.pmd.util.IOUtil;
+
+/**
+ * The Python tokenizer.
+ */
+public class PythonTokenizer implements Tokenizer {
+
+    @Override
+    public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
+        StringBuilder buffer = sourceCode.getCodeBuffer();
+        Reader reader = null;
+        try {
+            LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME)
+                    .getDefaultVersion().getLanguageVersionHandler();
+            reader = new StringReader(buffer.toString());
+            reader = IOUtil.skipBOM(reader);
+            TokenManager tokenManager = languageVersionHandler
+                    .getParser(languageVersionHandler.getDefaultParserOptions())
+                    .getTokenManager(sourceCode.getFileName(), reader);
+            Token currentToken = (Token) tokenManager.getNextToken();
+            while (currentToken.image.length() > 0) {
+                tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
+                currentToken = (Token) tokenManager.getNextToken();
+            }
+            tokenEntries.add(TokenEntry.getEOF());
+            System.err.println("Added " + sourceCode);
+        } catch (TokenMgrError err) {
+            err.printStackTrace();
+            System.err.println("Skipping " + sourceCode + " due to parse error");
+            tokenEntries.add(TokenEntry.getEOF());
+        } finally {
+            IOUtils.closeQuietly(reader);
+        }
+    }
+}

pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonHandler.java

@@ -1,26 +1,26 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.python;
-
-import net.sourceforge.pmd.lang.AbstractLanguageVersionHandler;
-import net.sourceforge.pmd.lang.Parser;
-import net.sourceforge.pmd.lang.ParserOptions;
-import net.sourceforge.pmd.lang.rule.RuleViolationFactory;
-
-/**
- * Implementation of LanguageVersionHandler for the Python Language.
- */
-public class PythonHandler extends AbstractLanguageVersionHandler {
-
-    @Override
-    public RuleViolationFactory getRuleViolationFactory() {
-        throw new UnsupportedOperationException("getRuleViolationFactory() is not supported for Python");
-    }
-
-    @Override
-    public Parser getParser(ParserOptions parserOptions) {
-        return new PythonParser(parserOptions);
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.python;
+
+import net.sourceforge.pmd.lang.AbstractLanguageVersionHandler;
+import net.sourceforge.pmd.lang.Parser;
+import net.sourceforge.pmd.lang.ParserOptions;
+import net.sourceforge.pmd.lang.rule.RuleViolationFactory;
+
+/**
+ * Implementation of LanguageVersionHandler for the Python Language.
+ */
+public class PythonHandler extends AbstractLanguageVersionHandler {
+
+    @Override
+    public RuleViolationFactory getRuleViolationFactory() {
+        throw new UnsupportedOperationException("getRuleViolationFactory() is not supported for Python");
+    }
+
+    @Override
+    public Parser getParser(ParserOptions parserOptions) {
+        return new PythonParser(parserOptions);
+    }
+}

pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonLanguageModule.java

@@ -1,27 +1,27 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.python;
-
-import net.sourceforge.pmd.lang.BaseLanguageModule;
-
-/**
- * Implementation of the Python Language Module.
- */
-public class PythonLanguageModule extends BaseLanguageModule {
-
-    /** The name, that can be used to display the language in UI. */
-    public static final String NAME = "Python";
-    /** The internal name. */
-    public static final String TERSE_NAME = "python";
-
-    /**
-     * Creates a new instance of {@link PythonLanguageModule} with the default
-     * file extensions for Python.
-     */
-    public PythonLanguageModule() {
-        super(NAME, null, TERSE_NAME, null, "py");
-        addVersion("", new PythonHandler(), true);
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.python;
+
+import net.sourceforge.pmd.lang.BaseLanguageModule;
+
+/**
+ * Implementation of the Python Language Module.
+ */
+public class PythonLanguageModule extends BaseLanguageModule {
+
+    /** The name, that can be used to display the language in UI. */
+    public static final String NAME = "Python";
+    /** The internal name. */
+    public static final String TERSE_NAME = "python";
+
+    /**
+     * Creates a new instance of {@link PythonLanguageModule} with the default
+     * file extensions for Python.
+     */
+    public PythonLanguageModule() {
+        super(NAME, null, TERSE_NAME, null, "py");
+        addVersion("", new PythonHandler(), true);
+    }
+}

pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonParser.java

@@ -1,52 +1,52 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.python;
-
-import java.io.Reader;
-import java.util.Map;
-
-import net.sourceforge.pmd.lang.AbstractParser;
-import net.sourceforge.pmd.lang.ParserOptions;
-import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.AbstractTokenManager;
-import net.sourceforge.pmd.lang.ast.Node;
-import net.sourceforge.pmd.lang.ast.ParseException;
-
-/**
- * Adapter for the Python Parser.
- */
-public class PythonParser extends AbstractParser {
-
-    /**
-     * Creates a new Python Parser.
-     * 
-     * @param parserOptions
-     *            the options
-     */
-    public PythonParser(ParserOptions parserOptions) {
-        super(parserOptions);
-    }
-
-    @Override
-    public TokenManager createTokenManager(Reader source) {
-        return new PythonTokenManager(source);
-    }
-
-    @Override
-    public boolean canParse() {
-        return false;
-    }
-
-    @Override
-    public Node parse(String fileName, Reader source) throws ParseException {
-        AbstractTokenManager.setFileName(fileName);
-        throw new UnsupportedOperationException("parse(Reader) is not supported for Python");
-    }
-
-    @Override
-    public Map<Integer, String> getSuppressMap() {
-        throw new UnsupportedOperationException("getSuppressMap() is not supported for Python");
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.python;
+
+import java.io.Reader;
+import java.util.Map;
+
+import net.sourceforge.pmd.lang.AbstractParser;
+import net.sourceforge.pmd.lang.ParserOptions;
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.AbstractTokenManager;
+import net.sourceforge.pmd.lang.ast.Node;
+import net.sourceforge.pmd.lang.ast.ParseException;
+
+/**
+ * Adapter for the Python Parser.
+ */
+public class PythonParser extends AbstractParser {
+
+    /**
+     * Creates a new Python Parser.
+     * 
+     * @param parserOptions
+     *            the options
+     */
+    public PythonParser(ParserOptions parserOptions) {
+        super(parserOptions);
+    }
+
+    @Override
+    public TokenManager createTokenManager(Reader source) {
+        return new PythonTokenManager(source);
+    }
+
+    @Override
+    public boolean canParse() {
+        return false;
+    }
+
+    @Override
+    public Node parse(String fileName, Reader source) throws ParseException {
+        AbstractTokenManager.setFileName(fileName);
+        throw new UnsupportedOperationException("parse(Reader) is not supported for Python");
+    }
+
+    @Override
+    public Map<Integer, String> getSuppressMap() {
+        throw new UnsupportedOperationException("getSuppressMap() is not supported for Python");
+    }
+}

pmd-python/src/main/java/net/sourceforge/pmd/lang/python/PythonTokenManager.java

@@ -1,37 +1,37 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.lang.python;
-
-import java.io.Reader;
-
-import net.sourceforge.pmd.lang.TokenManager;
-import net.sourceforge.pmd.lang.ast.SimpleCharStream;
-import net.sourceforge.pmd.lang.python.ast.PythonParserTokenManager;
-
-/**
- * Python Token Manager implementation.
- */
-public class PythonTokenManager implements TokenManager {
-    private final PythonParserTokenManager tokenManager;
-
-    /**
-     * Creates a new Python Token Manager from the given source code.
-     * 
-     * @param source
-     *            the source code
-     */
-    public PythonTokenManager(Reader source) {
-        tokenManager = new PythonParserTokenManager(new SimpleCharStream(source));
-    }
-
-    public Object getNextToken() {
-        return tokenManager.getNextToken();
-    }
-
-    @Override
-    public void setFileName(String fileName) {
-        PythonParserTokenManager.setFileName(fileName);
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.python;
+
+import java.io.Reader;
+
+import net.sourceforge.pmd.lang.TokenManager;
+import net.sourceforge.pmd.lang.ast.SimpleCharStream;
+import net.sourceforge.pmd.lang.python.ast.PythonParserTokenManager;
+
+/**
+ * Python Token Manager implementation.
+ */
+public class PythonTokenManager implements TokenManager {
+    private final PythonParserTokenManager tokenManager;
+
+    /**
+     * Creates a new Python Token Manager from the given source code.
+     * 
+     * @param source
+     *            the source code
+     */
+    public PythonTokenManager(Reader source) {
+        tokenManager = new PythonParserTokenManager(new SimpleCharStream(source));
+    }
+
+    public Object getNextToken() {
+        return tokenManager.getNextToken();
+    }
+
+    @Override
+    public void setFileName(String fileName) {
+        PythonParserTokenManager.setFileName(fileName);
+    }
+}

pmd-python/src/test/java/net/sourceforge/pmd/LanguageVersionDiscovererTest.java

@@ -1,32 +1,32 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.junit.Test;
-
-import net.sourceforge.pmd.lang.LanguageRegistry;
-import net.sourceforge.pmd.lang.LanguageVersion;
-import net.sourceforge.pmd.lang.LanguageVersionDiscoverer;
-import net.sourceforge.pmd.lang.python.PythonLanguageModule;
-
-public class LanguageVersionDiscovererTest {
-
-    /**
-     * Test on Python file with default version
-     */
-    @Test
-    public void testPython() {
-        LanguageVersionDiscoverer discoverer = new LanguageVersionDiscoverer();
-        File pythonFile = new File("/path/to/MY_PACKAGE.py");
-
-        LanguageVersion languageVersion = discoverer.getDefaultLanguageVersionForFile(pythonFile);
-        assertEquals("LanguageVersion must be Python!",
-                LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion(), languageVersion);
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.LanguageVersion;
+import net.sourceforge.pmd.lang.LanguageVersionDiscoverer;
+import net.sourceforge.pmd.lang.python.PythonLanguageModule;
+
+public class LanguageVersionDiscovererTest {
+
+    /**
+     * Test on Python file with default version
+     */
+    @Test
+    public void testPython() {
+        LanguageVersionDiscoverer discoverer = new LanguageVersionDiscoverer();
+        File pythonFile = new File("/path/to/MY_PACKAGE.py");
+
+        LanguageVersion languageVersion = discoverer.getDefaultLanguageVersionForFile(pythonFile);
+        assertEquals("LanguageVersion must be Python!",
+                LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion(), languageVersion);
+    }
+}

pmd-python/src/test/java/net/sourceforge/pmd/LanguageVersionTest.java

@@ -1,27 +1,27 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.junit.runners.Parameterized.Parameters;
-
-import net.sourceforge.pmd.lang.LanguageRegistry;
-import net.sourceforge.pmd.lang.LanguageVersion;
-import net.sourceforge.pmd.lang.python.PythonLanguageModule;
-
-public class LanguageVersionTest extends AbstractLanguageVersionTest {
-
-    public LanguageVersionTest(String name, String terseName, String version, LanguageVersion expected) {
-        super(name, terseName, version, expected);
-    }
-
-    @Parameters
-    public static Collection<Object[]> data() {
-        return Arrays.asList(new Object[][] { { PythonLanguageModule.NAME, PythonLanguageModule.TERSE_NAME, "",
-            LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion(), }, });
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runners.Parameterized.Parameters;
+
+import net.sourceforge.pmd.lang.LanguageRegistry;
+import net.sourceforge.pmd.lang.LanguageVersion;
+import net.sourceforge.pmd.lang.python.PythonLanguageModule;
+
+public class LanguageVersionTest extends AbstractLanguageVersionTest {
+
+    public LanguageVersionTest(String name, String terseName, String version, LanguageVersion expected) {
+        super(name, terseName, version, expected);
+    }
+
+    @Parameters
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][] { { PythonLanguageModule.NAME, PythonLanguageModule.TERSE_NAME, "",
+            LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion(), }, });
+    }
+}

pmd-python/src/test/java/net/sourceforge/pmd/cpd/PythonTokenizerTest.java

@@ -1,36 +1,36 @@
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-
-package net.sourceforge.pmd.cpd;
-
-import java.io.IOException;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
-
-public class PythonTokenizerTest extends AbstractTokenizerTest {
-
-    private static final String FILENAME = "sample-python.py";
-
-    @Before
-    @Override
-    public void buildTokenizer() throws IOException {
-        this.tokenizer = new PythonTokenizer();
-        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
-    }
-
-    @Override
-    public String getSampleCode() throws IOException {
-        return IOUtils.toString(PythonTokenizer.class.getResourceAsStream(FILENAME));
-    }
-
-    @Test
-    public void tokenizeTest() throws IOException {
-        this.expectedTokenCount = 1218;
-        super.tokenizeTest();
-    }
-}
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.cpd;
+
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import net.sourceforge.pmd.testframework.AbstractTokenizerTest;
+
+public class PythonTokenizerTest extends AbstractTokenizerTest {
+
+    private static final String FILENAME = "sample-python.py";
+
+    @Before
+    @Override
+    public void buildTokenizer() throws IOException {
+        this.tokenizer = new PythonTokenizer();
+        this.sourceCode = new SourceCode(new SourceCode.StringCodeLoader(this.getSampleCode(), FILENAME));
+    }
+
+    @Override
+    public String getSampleCode() throws IOException {
+        return IOUtils.toString(PythonTokenizer.class.getResourceAsStream(FILENAME));
+    }
+
+    @Test
+    public void tokenizeTest() throws IOException {
+        this.expectedTokenCount = 1218;
+        super.tokenizeTest();
+    }
+}