diff --git a/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md b/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md index 7ba812383b..02b760aeb9 100644 --- a/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md +++ b/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md @@ -5,6 +5,7 @@ keywords: [formats, renderers] summary: "Overview of the built-in report formats for CPD" permalink: pmd_userdocs_cpd_report_formats.html author: Andreas Dangel +last_updated: June 2024 (7.3.0) --- ## Overview @@ -97,11 +98,19 @@ Starting at line 110 of /home/pmd/source/pmd-core/src/test/java/net/sourceforge/ This format uses XML to output the duplications in a more structured format. The XML format can then further be processed using XSLT transformations. See [section xslt](#xslt) for examples. +Since PMD 7.3.0 any recoverable errors are also reported as additional elements `error` to help investigate +any errors occurred during analysis. + Example: ```xml - + @@ -167,6 +176,32 @@ Example: Assert.assertEquals(2, query.nodeNameToXPaths.size()); Assert.assertEquals("self::node()[(attribute::Test1 = \"false\")][(attribute::Test2 = \"true\")]", query.nodeNameToXPaths.get("dummyNode").get(0).toString());]]> + net.sourceforge.pmd.lang.ast.LexException: Lexical error in file '/home/pmd/source/pmd-cli/src/test/resources/net/sourceforge/pmd/cli/cpd/badandgood/BadFile.java' at line 4, column 14: "\ufffd" (65533), after : "" (in lexical state DEFAULT) + at net.sourceforge.pmd.lang.ast.InternalApiBridge.newLexException(InternalApiBridge.java:25) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:2698) + at net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.getNextToken(JavaParserImplTokenManager.java:18) + at net.sourceforge.pmd.cpd.impl.BaseTokenFilter.getNextToken(BaseTokenFilter.java:44) + at net.sourceforge.pmd.cpd.impl.CpdLexerBase.tokenize(CpdLexerBase.java:40) + at net.sourceforge.pmd.cpd.CpdLexer.tokenize(CpdLexer.java:29) + at net.sourceforge.pmd.cpd.CpdAnalysis.doTokenize(CpdAnalysis.java:146) + at net.sourceforge.pmd.cpd.CpdAnalysis.performAnalysis(CpdAnalysis.java:173) + at net.sourceforge.pmd.cli.commands.internal.CpdCommand.doExecute(CpdCommand.java:134) + at net.sourceforge.pmd.cli.commands.internal.CpdCommand.doExecute(CpdCommand.java:29) + at net.sourceforge.pmd.cli.internal.PmdRootLogger.executeInLoggingContext(PmdRootLogger.java:55) + at net.sourceforge.pmd.cli.commands.internal.AbstractAnalysisPmdSubcommand.execute(AbstractAnalysisPmdSubcommand.java:111) + at net.sourceforge.pmd.cli.commands.internal.AbstractPmdSubcommand.call(AbstractPmdSubcommand.java:30) + at net.sourceforge.pmd.cli.commands.internal.AbstractPmdSubcommand.call(AbstractPmdSubcommand.java:16) + at picocli.CommandLine.executeUserObject(CommandLine.java:2041) + at picocli.CommandLine.access$1500(CommandLine.java:148) + at picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2461) + at picocli.CommandLine$RunLast.handle(CommandLine.java:2453) + at picocli.CommandLine$RunLast.handle(CommandLine.java:2415) + at picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2273) + at picocli.CommandLine$RunLast.execute(CommandLine.java:2417) + at picocli.CommandLine.execute(CommandLine.java:2170) + at net.sourceforge.pmd.cli.PmdCli.main(PmdCli.java:24) + ``` diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index aee8e21a14..b81538f603 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -15,6 +15,8 @@ This is a {{ site.pmd.release_type }} release. ### 🚀 New and noteworthy ### 🐛 Fixed Issues +* core + * [#4992](https://github.com/pmd/pmd/pull/4992): \[core] CPD: Include processing errors in XML report * apex * [#5053](https://github.com/pmd/pmd/issues/5053): \[apex] CPD fails to parse string literals with escaped characters * java-bestpractices @@ -25,6 +27,29 @@ This is a {{ site.pmd.release_type }} release. ### 🚨 API Changes +#### CPD Report Format XML + +There are some important changes: + +1. The XML format will now use an XSD schema, that is available at . + This schema defines the valid elements and attributes that one can expect from a CPD report. +2. The root element `pmd-cpd` contains the new attributes `pmdVersion`, `timestamp` and `version`. The latter is + the schema version and is currently "1.0.0". +3. The CPD XML report will now also contain recoverable errors as additional `` elements. + +See [Report formats for CPD](pmd_userdocs_cpd_report_formats.html#xml) for an example. + +The XML format should be compatible as only attributes and elements have been added. However, if you parse +the document with a namespace aware parser, you might encounter some issues like no elements being found. +In case the new format doesn't work for you (e.g. namespaces, unexpected error elements), you can +go back using the old format with the renderer "xmlold" ({%jdoc core::cpd.XMLOldRenderer %}). Note, that +this old renderer is deprecated and only there for compatibility reasons. Whatever tooling is used to +read the XML format should be updated. + +#### Deprecated for removal + +* {%jdoc !!core::cpd.XMLOldRenderer %} (the CPD format "xmlold"). + ### ✨ External Contributions {% endtocmaker %} diff --git a/pmd-ant/src/main/java/net/sourceforge/pmd/ant/CPDTask.java b/pmd-ant/src/main/java/net/sourceforge/pmd/ant/CPDTask.java index 607186cb8a..58bf8967d5 100644 --- a/pmd-ant/src/main/java/net/sourceforge/pmd/ant/CPDTask.java +++ b/pmd-ant/src/main/java/net/sourceforge/pmd/ant/CPDTask.java @@ -29,6 +29,7 @@ import net.sourceforge.pmd.cpd.CPDReportRenderer; import net.sourceforge.pmd.cpd.CSVRenderer; import net.sourceforge.pmd.cpd.CpdAnalysis; import net.sourceforge.pmd.cpd.SimpleRenderer; +import net.sourceforge.pmd.cpd.XMLOldRenderer; import net.sourceforge.pmd.cpd.XMLRenderer; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguageRegistry; @@ -66,6 +67,8 @@ public class CPDTask extends Task { private static final String TEXT_FORMAT = "text"; private static final String XML_FORMAT = "xml"; + @Deprecated + private static final String XMLOLD_FORMAT = "xmlold"; private static final String CSV_FORMAT = "csv"; private String format = TEXT_FORMAT; @@ -177,6 +180,8 @@ public class CPDTask extends Task { return new SimpleRenderer(); } else if (CSV_FORMAT.equals(format)) { return new CSVRenderer(); + } else if (XMLOLD_FORMAT.equals(format)) { + return new XMLOldRenderer(); } return new XMLRenderer(); } @@ -253,7 +258,7 @@ public class CPDTask extends Task { } public static class FormatAttribute extends EnumeratedAttribute { - private static final String[] FORMATS = new String[] { XML_FORMAT, TEXT_FORMAT, CSV_FORMAT }; + private static final String[] FORMATS = new String[] { XML_FORMAT, TEXT_FORMAT, CSV_FORMAT, XMLOLD_FORMAT }; @Override public String[] getValues() { diff --git a/pmd-cli/src/test/java/net/sourceforge/pmd/cli/CpdCliTest.java b/pmd-cli/src/test/java/net/sourceforge/pmd/cli/CpdCliTest.java index af1eee72aa..b8388af890 100644 --- a/pmd-cli/src/test/java/net/sourceforge/pmd/cli/CpdCliTest.java +++ b/pmd-cli/src/test/java/net/sourceforge/pmd/cli/CpdCliTest.java @@ -10,7 +10,6 @@ import static net.sourceforge.pmd.util.CollectionUtil.listOf; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import java.nio.charset.StandardCharsets; @@ -38,6 +37,14 @@ class CpdCliTest extends BaseCliTest { private static final String SRC_DIR = BASE_RES_PATH + "files/"; private static final Path SRC_PATH = Paths.get(SRC_DIR).toAbsolutePath(); + private static final String CPD_REPORT_HEADER_PATTERN = "<\\?xml version=\"1.0\" encoding=\"UTF-8\"\\?>\n" + + "\n"; + private static final Map NUMBER_OF_TOKENS; static { @@ -68,8 +75,11 @@ class CpdCliTest extends BaseCliTest { void testEmptyResultRendering() throws Exception { final String expectedFilesXml = getExpectedFileEntriesXml(NUMBER_OF_TOKENS.keySet()); runCliSuccessfully("--minimum-tokens", "340", "--language", "java", "--dir", SRC_DIR, "--format", "xml") - .verify(result -> result.checkStdOut(equalTo( - "" + "\n" + "\n" + expectedFilesXml + "\n" + .verify(result -> result.checkStdOut(containsPattern(CPD_REPORT_HEADER_PATTERN + + "\\Q" // quote start + + expectedFilesXml + + "\n" + + "\\E" // quote end ))); } @@ -176,8 +186,8 @@ class CpdCliTest extends BaseCliTest { @Test void testNoDuplicatesResultRendering() throws Exception { - String expectedReport = "\n" - + "\n" + String expectedReportPattern = CPD_REPORT_HEADER_PATTERN + + "\\Q" // quote start + " \n" + " \n" + " \n" - + "\n"; + + "\n" + + "\\E"; // quote end runCliSuccessfully("--minimum-tokens", "340", "--language", "java", "--dir", SRC_DIR, "--format", "xml") - .verify(result -> result.checkStdOut(equalTo(expectedReport))); + .verify(result -> result.checkStdOut(containsPattern(expectedReportPattern))); } /** @@ -251,9 +262,7 @@ class CpdCliTest extends BaseCliTest { runCli(OK, "--minimum-tokens", "5", "--language", "ecmascript", "-f", "xml", "-d", BASE_RES_PATH + "tsFiles/") - .checkStdOut(equalTo( - "\n\n" - )); + .checkStdOut(containsPattern(CPD_REPORT_HEADER_PATTERN.substring(0, CPD_REPORT_HEADER_PATTERN.length() - 2) + "/>")); } @Test diff --git a/pmd-compat6/src/main/java/net/sourceforge/pmd/cpd/RendererHelper.java b/pmd-compat6/src/main/java/net/sourceforge/pmd/cpd/RendererHelper.java index f7291e9ae5..c25a80d964 100644 --- a/pmd-compat6/src/main/java/net/sourceforge/pmd/cpd/RendererHelper.java +++ b/pmd-compat6/src/main/java/net/sourceforge/pmd/cpd/RendererHelper.java @@ -39,7 +39,7 @@ final class RendererHelper { } try (SourceManager sourceManager = new SourceManager(textFiles)) { - CPDReport report = new CPDReport(sourceManager, matchesList, Collections.emptyMap()); + CPDReport report = new CPDReport(sourceManager, matchesList, Collections.emptyMap(), Collections.emptyList()); renderer.render(report, writer); } catch (Exception e) { throw new RuntimeException(e); diff --git a/pmd-core/etc/xslt/cpdhtml-v2.xslt b/pmd-core/etc/xslt/cpdhtml-v2.xslt index c1862d9cba..614d0f793f 100644 --- a/pmd-core/etc/xslt/cpdhtml-v2.xslt +++ b/pmd-core/etc/xslt/cpdhtml-v2.xslt @@ -1,5 +1,7 @@ - + @@ -68,10 +70,10 @@ Approximate number of bytes - - - - + + + + @@ -91,7 +93,7 @@ - + @@ -133,7 +135,7 @@ - + @@ -141,7 +143,7 @@ - + @@ -152,7 +154,7 @@
columnendcolumnlineendlinepath
-
+
diff --git a/pmd-core/etc/xslt/cpdhtml.xslt b/pmd-core/etc/xslt/cpdhtml.xslt index ef6e9ac94f..97265d97c0 100644 --- a/pmd-core/etc/xslt/cpdhtml.xslt +++ b/pmd-core/etc/xslt/cpdhtml.xslt @@ -1,11 +1,13 @@ - + 30 - + @@ -46,10 +48,10 @@ Approx # bytes - - - - + + + +

@@ -58,13 +60,13 @@

- +
IDFilesLines
- +
../src/.html# line
@@ -82,7 +84,7 @@
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java index d97cc8068a..f29b64b8d6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java @@ -39,6 +39,7 @@ public class CPDConfiguration extends AbstractConfiguration { static { RENDERERS.put(DEFAULT_RENDERER, SimpleRenderer.class); RENDERERS.put("xml", XMLRenderer.class); + RENDERERS.put("xmlold", XMLOldRenderer.class); RENDERERS.put("csv", CSVRenderer.class); RENDERERS.put("csv_with_linecount_per_file", CSVWithLinecountPerFileRenderer.class); RENDERERS.put("vs", VSRenderer.class); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDReport.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDReport.java index 7278d4923e..50bbf6ccaa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDReport.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDReport.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.FileId; +import net.sourceforge.pmd.reporting.Report; /** * The result of a CPD analysis. This is rendered by a {@link CPDReportRenderer}. @@ -24,13 +25,16 @@ public class CPDReport { private final SourceManager sourceManager; private final List matches; private final Map numberOfTokensPerFile; + private final List processingErrors; CPDReport(SourceManager sourceManager, List matches, - Map numberOfTokensPerFile) { + Map numberOfTokensPerFile, + List processingErrors) { this.sourceManager = sourceManager; this.matches = Collections.unmodifiableList(matches); this.numberOfTokensPerFile = Collections.unmodifiableMap(new TreeMap<>(numberOfTokensPerFile)); + this.processingErrors = Collections.unmodifiableList(processingErrors); } /** Return the list of duplication matches found by the CPD analysis. */ @@ -39,11 +43,15 @@ public class CPDReport { } /** Return a map containing the number of tokens by processed file. */ - public Map getNumberOfTokensPerFile() { return numberOfTokensPerFile; } + /** Returns the list of occurred processing errors. */ + public List getProcessingErrors() { + return processingErrors; + } + /** * Return the slice of source code where the mark was found. This * returns the entire lines from the start to the end line of the @@ -66,7 +74,7 @@ public class CPDReport { public CPDReport filterMatches(Predicate filter) { List filtered = this.matches.stream().filter(filter).collect(Collectors.toList()); - return new CPDReport(sourceManager, filtered, this.getNumberOfTokensPerFile()); + return new CPDReport(sourceManager, filtered, this.getNumberOfTokensPerFile(), this.getProcessingErrors()); } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java index ccc2b6c686..457194166a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd; import java.io.IOException; import java.io.Writer; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +31,7 @@ import net.sourceforge.pmd.lang.document.InternalApiBridge; import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.document.TextFile; import net.sourceforge.pmd.properties.PropertyDescriptor; +import net.sourceforge.pmd.reporting.Report; import net.sourceforge.pmd.util.log.PmdReporter; /** @@ -162,7 +164,7 @@ public final class CpdAnalysis implements AutoCloseable { Map numberOfTokensPerFile = new HashMap<>(); - boolean hasErrors = false; + List processingErrors = new ArrayList<>(); Tokens tokens = new Tokens(); for (TextFile textFile : sourceManager.getTextFiles()) { TextDocument textDocument = sourceManager.get(textFile); @@ -177,11 +179,11 @@ public final class CpdAnalysis implements AutoCloseable { } String message = configuration.isSkipLexicalErrors() ? "Skipping file" : "Error while tokenizing"; reporter.errorEx(message, e); - hasErrors = true; + processingErrors.add(new Report.ProcessingError(e, textFile.getFileId())); savedState.restore(tokens); } } - if (hasErrors && !configuration.isSkipLexicalErrors()) { + if (!processingErrors.isEmpty() && !configuration.isSkipLexicalErrors()) { // will be caught by CPD command throw new IllegalStateException("Errors were detected while lexing source, exiting because --skip-lexical-errors is unset."); } @@ -192,7 +194,7 @@ public final class CpdAnalysis implements AutoCloseable { tokens = null; // NOPMD null it out before rendering LOGGER.debug("Finished: {} duplicates found", matches.size()); - CPDReport cpdReport = new CPDReport(sourceManager, matches, numberOfTokensPerFile); + CPDReport cpdReport = new CPDReport(sourceManager, matches, numberOfTokensPerFile, processingErrors); if (renderer != null) { try (Writer writer = IOUtil.createWriter(Charset.defaultCharset(), null)) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java index 9231711559..91e1bb8fe6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java @@ -267,7 +267,7 @@ public class GUI implements CPDListener { } if (!f.canWrite()) { - final CPDReport report = new CPDReport(sourceManager, matches, numberOfTokensPerFile); + final CPDReport report = new CPDReport(sourceManager, matches, numberOfTokensPerFile, Collections.emptyList()); try (PrintWriter pw = new PrintWriter(Files.newOutputStream(f.toPath()))) { renderer.render(report, pw); pw.flush(); @@ -549,7 +549,7 @@ public class GUI implements CPDListener { for (int selectionIndex : selectionIndices) { selections.add((Match) model.getValueAt(selectionIndex, 99)); } - CPDReport toRender = new CPDReport(sourceManager, selections, Collections.emptyMap()); + CPDReport toRender = new CPDReport(sourceManager, selections, Collections.emptyMap(), Collections.emptyList()); String report = new SimpleRenderer(trimLeadingWhitespace).renderToString(toRender); resultsTextArea.setText(report); resultsTextArea.setCaretPosition(0); // move to the top diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLOldRenderer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLOldRenderer.java new file mode 100644 index 0000000000..d5867e1399 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLOldRenderer.java @@ -0,0 +1,34 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import java.io.IOException; +import java.io.Writer; + +/** + * Provides backwards compatible XML renderer, which doesn't use namespaces, schema and + * doesn't output error information. + * + *

This renderer is available as "xmlold". + * + * @deprecated Update your tools to use the standard XML renderer "xml" again. + */ +@Deprecated +public class XMLOldRenderer implements CPDReportRenderer { + private final XMLRenderer xmlRenderer; + + public XMLOldRenderer() { + this(null); + } + + public XMLOldRenderer(String encoding) { + this.xmlRenderer = new XMLRenderer(encoding, false); + } + + @Override + public void render(CPDReport report, Writer writer) throws IOException { + xmlRenderer.render(report, writer); + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java index 5365d1827a..8e9b8fe5b6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java @@ -6,7 +6,10 @@ package net.sourceforge.pmd.cpd; import java.io.IOException; import java.io.Writer; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.Map; +import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -20,9 +23,11 @@ import javax.xml.transform.stream.StreamResult; import org.w3c.dom.Document; import org.w3c.dom.Element; +import net.sourceforge.pmd.PMDVersion; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.FileId; import net.sourceforge.pmd.lang.document.FileLocation; +import net.sourceforge.pmd.reporting.Report; import net.sourceforge.pmd.util.StringUtil; /** @@ -31,14 +36,19 @@ import net.sourceforge.pmd.util.StringUtil; * */ public final class XMLRenderer implements CPDReportRenderer { + private static final String NAMESPACE_URI = "https://pmd-code.org/schema/cpd-report"; + private static final String NAMESPACE_LOCATION = "https://pmd.github.io/schema/cpd-report_1_0_0.xsd"; + private static final String SCHEMA_VERSION = "1.0.0"; private String encoding; + private final boolean newFormat; + /** * Creates a XML Renderer with the default (platform dependent) encoding. */ public XMLRenderer() { - this(null); + this(null, true); } /** @@ -49,7 +59,12 @@ public final class XMLRenderer implements CPDReportRenderer { * dependent) encoding is used. */ public XMLRenderer(String encoding) { + this(encoding, true); + } + + XMLRenderer(String encoding, boolean newFormat) { setEncoding(encoding); + this.newFormat = newFormat; } public void setEncoding(String encoding) { @@ -82,7 +97,11 @@ public final class XMLRenderer implements CPDReportRenderer { transformer.setOutputProperty(OutputKeys.METHOD, "xml"); transformer.setOutputProperty(OutputKeys.ENCODING, encoding); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty(OutputKeys.CDATA_SECTION_ELEMENTS, "codefragment"); + if (newFormat) { + transformer.setOutputProperty(OutputKeys.CDATA_SECTION_ELEMENTS, "{" + NAMESPACE_URI + "}codefragment"); + } else { + transformer.setOutputProperty(OutputKeys.CDATA_SECTION_ELEMENTS, "codefragment"); + } transformer.transform(new DOMSource(doc), new StreamResult(writer)); } catch (TransformerException e) { throw new IllegalStateException(e); @@ -93,14 +112,22 @@ public final class XMLRenderer implements CPDReportRenderer { @Override public void render(final CPDReport report, final Writer writer) throws IOException { final Document doc = createDocument(); - final Element root = doc.createElement("pmd-cpd"); + final Element root = createElement(doc, "pmd-cpd"); + + if (newFormat) { + root.setAttributeNS(XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "xsi:schemaLocation", NAMESPACE_URI + " " + NAMESPACE_LOCATION); + root.setAttributeNS(NAMESPACE_URI, "version", SCHEMA_VERSION); + root.setAttributeNS(NAMESPACE_URI, "pmdVersion", PMDVersion.VERSION); + root.setAttributeNS(NAMESPACE_URI, "timestamp", OffsetDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + final Map numberOfTokensPerFile = report.getNumberOfTokensPerFile(); doc.appendChild(root); for (final Map.Entry pair : numberOfTokensPerFile.entrySet()) { - final Element fileElement = doc.createElement("file"); - fileElement.setAttribute("path", report.getDisplayName(pair.getKey())); - fileElement.setAttribute("totalNumberOfTokens", String.valueOf(pair.getValue())); + final Element fileElement = createElement(doc, "file"); + setAttribute(fileElement, "path", report.getDisplayName(pair.getKey())); + setAttribute(fileElement, "totalNumberOfTokens", String.valueOf(pair.getValue())); root.appendChild(fileElement); } @@ -110,23 +137,34 @@ public final class XMLRenderer implements CPDReportRenderer { addCodeSnippet(doc, dupElt, match, report); root.appendChild(dupElt); } + + if (newFormat) { + for (Report.ProcessingError error : report.getProcessingErrors()) { + Element errorElt = doc.createElementNS(NAMESPACE_URI, "error"); + errorElt.setAttributeNS(NAMESPACE_URI, "filename", report.getDisplayName(error.getFileId())); + errorElt.setAttributeNS(NAMESPACE_URI, "msg", error.getMsg()); + errorElt.setTextContent(error.getDetail()); + root.appendChild(errorElt); + } + } + dumpDocToWriter(doc, writer); writer.flush(); } private void addFilesToDuplicationElement(Document doc, Element duplication, Match match, CPDReport report) { for (Mark mark : match) { - final Element file = doc.createElement("file"); + final Element file = createElement(doc, "file"); FileLocation loc = mark.getLocation(); - file.setAttribute("line", String.valueOf(loc.getStartLine())); + setAttribute(file, "line", String.valueOf(loc.getStartLine())); // only remove invalid characters, escaping is done by the DOM impl. String filenameXml10 = StringUtil.removedInvalidXml10Characters(report.getDisplayName(loc.getFileId())); - file.setAttribute("path", filenameXml10); - file.setAttribute("endline", String.valueOf(loc.getEndLine())); - file.setAttribute("column", String.valueOf(loc.getStartColumn())); - file.setAttribute("endcolumn", String.valueOf(loc.getEndColumn())); - file.setAttribute("begintoken", String.valueOf(mark.getBeginTokenIndex())); - file.setAttribute("endtoken", String.valueOf(mark.getEndTokenIndex())); + setAttribute(file, "path", filenameXml10); + setAttribute(file, "endline", String.valueOf(loc.getEndLine())); + setAttribute(file, "column", String.valueOf(loc.getStartColumn())); + setAttribute(file, "endcolumn", String.valueOf(loc.getEndColumn())); + setAttribute(file, "begintoken", String.valueOf(mark.getBeginTokenIndex())); + setAttribute(file, "endtoken", String.valueOf(mark.getEndTokenIndex())); duplication.appendChild(file); } } @@ -136,7 +174,7 @@ public final class XMLRenderer implements CPDReportRenderer { if (codeSnippet != null) { // the code snippet has normalized line endings String platformSpecific = codeSnippet.toString().replace("\n", System.lineSeparator()); - Element codefragment = doc.createElement("codefragment"); + Element codefragment = createElement(doc, "codefragment"); // only remove invalid characters, escaping is not necessary in CDATA. // if the string contains the end marker of a CDATA section, then the DOM impl will // create two cdata sections automatically. @@ -146,9 +184,24 @@ public final class XMLRenderer implements CPDReportRenderer { } private Element createDuplicationElement(Document doc, Match match) { - Element duplication = doc.createElement("duplication"); - duplication.setAttribute("lines", String.valueOf(match.getLineCount())); - duplication.setAttribute("tokens", String.valueOf(match.getTokenCount())); + Element duplication = createElement(doc, "duplication"); + setAttribute(duplication, "lines", String.valueOf(match.getLineCount())); + setAttribute(duplication, "tokens", String.valueOf(match.getTokenCount())); return duplication; } + + private Element createElement(Document doc, String name) { + if (newFormat) { + return doc.createElementNS(NAMESPACE_URI, name); + } + return doc.createElement(name); + } + + private void setAttribute(Element element, String name, String value) { + if (newFormat) { + element.setAttributeNS(NAMESPACE_URI, name, value); + } else { + element.setAttribute(name, value); + } + } } diff --git a/pmd-core/src/main/resources/cpd-report_1_0_0.xsd b/pmd-core/src/main/resources/cpd-report_1_0_0.xsd new file mode 100644 index 0000000000..71745eda95 --- /dev/null +++ b/pmd-core/src/main/resources/cpd-report_1_0_0.xsd @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdAnalysisTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdAnalysisTest.java index 84c22ffc6c..e5140e9abe 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdAnalysisTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdAnalysisTest.java @@ -4,8 +4,11 @@ package net.sourceforge.pmd.cpd; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.any; @@ -22,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -34,6 +38,7 @@ import net.sourceforge.pmd.lang.ast.LexException; import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.FileId; import net.sourceforge.pmd.lang.document.TextFile; +import net.sourceforge.pmd.reporting.Report; import net.sourceforge.pmd.util.log.PmdReporter; /** @@ -212,6 +217,36 @@ class CpdAnalysisTest { verifyNoMoreInteractions(reporter); } + @Test + void reportShouldContainProcessingErrors() throws IOException { + AtomicReference report = new AtomicReference<>(); + PmdReporter reporter = mock(PmdReporter.class); + config.setReporter(reporter); + + config.setSkipLexicalErrors(true); // must be true, otherwise CPD is aborted with first processing error + try (CpdAnalysis cpd = CpdAnalysis.create(config)) { + assertTrue(cpd.files().addSourceFile(FileId.fromPathLikeString("foo.dummy"), DummyLanguageModule.CPD_THROW_LEX_EXCEPTION)); + assertTrue(cpd.files().addSourceFile(FileId.fromPathLikeString("foo2.dummy"), DummyLanguageModule.CPD_THROW_MALFORMED_SOURCE_EXCEPTION)); + cpd.performAnalysis(report::set); + } + + assertNotNull(report.get(), "CPD aborted early without producing a report"); + List processingErrors = report.get().getProcessingErrors(); + assertEquals(2, processingErrors.size()); + + Report.ProcessingError error1 = processingErrors.get(0); + assertEquals("foo.dummy", error1.getFileId().getFileName()); + assertThat(error1.getDetail(), containsString(LexException.class.getSimpleName())); + + Report.ProcessingError error2 = processingErrors.get(1); + assertEquals("foo2.dummy", error2.getFileId().getFileName()); + assertThat(error2.getDetail(), containsString(MalformedSourceException.class.getSimpleName())); + + verify(reporter).errorEx(eq("Skipping file"), any(LexException.class)); + verify(reporter).errorEx(eq("Skipping file"), any(MalformedSourceException.class)); + verifyNoMoreInteractions(reporter); + } + @Test void testSkipLexicalErrors() throws IOException { PmdReporter reporter = mock(PmdReporter.class); diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdTestUtils.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdTestUtils.java index e40b4c34c7..1a41f61666 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdTestUtils.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CpdTestUtils.java @@ -15,6 +15,7 @@ import java.util.Set; import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.document.FileId; import net.sourceforge.pmd.lang.document.TextFile; +import net.sourceforge.pmd.reporting.Report; final class CpdTestUtils { @@ -26,10 +27,10 @@ final class CpdTestUtils { } static CPDReport makeReport(List matches) { - return makeReport(matches, Collections.emptyMap()); + return makeReport(matches, Collections.emptyMap(), Collections.emptyList()); } - static CPDReport makeReport(List matches, Map numTokensPerFile) { + static CPDReport makeReport(List matches, Map numTokensPerFile, List processingErrors) { Set textFiles = new HashSet<>(); for (Match match : matches) { match.iterator().forEachRemaining( @@ -41,7 +42,8 @@ final class CpdTestUtils { return new CPDReport( new SourceManager(new ArrayList<>(textFiles)), matches, - numTokensPerFile + numTokensPerFile, + processingErrors ); } @@ -73,7 +75,8 @@ final class CpdTestUtils { return new CPDReport( new SourceManager(new ArrayList<>(textFiles)), matches, - numTokensPerFile + numTokensPerFile, + Collections.emptyList() ); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLOldRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLOldRendererTest.java new file mode 100644 index 0000000000..3c0c74b340 --- /dev/null +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLOldRendererTest.java @@ -0,0 +1,88 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import net.sourceforge.pmd.lang.document.FileId; + +class XMLOldRendererTest { + private static final String ENCODING = (String) System.getProperties().get("file.encoding"); + + @Test + void testWithNoDuplication() throws IOException, ParserConfigurationException, SAXException { + CPDReportRenderer renderer = new XMLOldRenderer(); + StringWriter sw = new StringWriter(); + renderer.render(CpdTestUtils.makeReport(Collections.emptyList()), sw); + String report = sw.toString(); + + assertEquals("\n\n", + report, + "no namespace expected"); + + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); + NodeList nodes = doc.getChildNodes(); + Node n = nodes.item(0); + assertEquals("pmd-cpd", n.getNodeName()); + assertEquals(0, doc.getElementsByTagName("duplication").getLength()); + } + + @Test + void testWithOneDuplication() throws Exception { + CPDReportRenderer renderer = new XMLOldRenderer(); + CpdTestUtils.CpdReportBuilder builder = new CpdTestUtils.CpdReportBuilder(); + int lineCount = 6; + FileId foo1 = CpdTestUtils.FOO_FILE_ID; + Mark mark1 = builder.createMark("public", foo1, 1, lineCount); + Mark mark2 = builder.createMark("stuff", foo1, 73, lineCount); + builder.addMatch(new Match(75, mark1, mark2)); + + StringWriter sw = new StringWriter(); + renderer.render(builder.build(), sw); + String report = sw.toString(); + + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); + NodeList dupes = doc.getElementsByTagName("duplication"); + assertEquals(1, dupes.getLength()); + Node file = dupes.item(0).getFirstChild(); + while (file != null && file.getNodeType() != Node.ELEMENT_NODE) { + file = file.getNextSibling(); + } + if (file != null) { + assertEquals("1", file.getAttributes().getNamedItem("line").getNodeValue()); + assertEquals(foo1.getAbsolutePath(), file.getAttributes().getNamedItem("path").getNodeValue()); + assertEquals("6", file.getAttributes().getNamedItem("endline").getNodeValue()); + assertEquals("1", file.getAttributes().getNamedItem("column").getNodeValue()); + assertEquals("1", file.getAttributes().getNamedItem("endcolumn").getNodeValue()); + file = file.getNextSibling(); + while (file != null && file.getNodeType() != Node.ELEMENT_NODE) { + file = file.getNextSibling(); + } + } + if (file != null) { + assertEquals("73", file.getAttributes().getNamedItem("line").getNodeValue()); + assertEquals("78", file.getAttributes().getNamedItem("endline").getNodeValue()); + assertEquals("1", file.getAttributes().getNamedItem("column").getNodeValue()); + assertEquals("1", file.getAttributes().getNamedItem("endcolumn").getNodeValue()); + } + assertEquals(1, doc.getElementsByTagName("codefragment").getLength()); + assertEquals(CpdTestUtils.generateDummyContent(lineCount), doc.getElementsByTagName("codefragment").item(0).getTextContent()); + } +} diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java index eb96c32af9..b22dc67307 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java @@ -11,20 +11,32 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.StringReader; import java.io.StringWriter; import java.util.Collections; +import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import org.xml.sax.helpers.DefaultHandler; import net.sourceforge.pmd.cpd.CpdTestUtils.CpdReportBuilder; +import net.sourceforge.pmd.lang.ast.LexException; import net.sourceforge.pmd.lang.document.FileId; +import net.sourceforge.pmd.reporting.Report; /** * @author Philippe T'Seyen @@ -42,6 +54,18 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(CpdTestUtils.makeReport(Collections.emptyList()), sw); String report = sw.toString(); + assertReportIsValidSchema(report); + + assertEquals("\n" + + "\n", + report.replaceAll("timestamp=\".+?\"", "timestamp=\"XXX\"") + .replaceAll("pmdVersion=\".+?\"", "pmdVersion=\"XXX\""), + "namespace is missing or wrong"); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); @@ -64,6 +88,7 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(builder.build(), sw); String report = sw.toString(); + assertReportIsValidSchema(report); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); @@ -113,6 +138,7 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(builder.build(), sw); String report = sw.toString(); + assertReportIsValidSchema(report); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); @@ -133,6 +159,7 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(builder.build(), sw); String report = sw.toString(); + assertReportIsValidSchema(report); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); @@ -164,7 +191,7 @@ class XMLRendererTest { } @Test - void testRendererEncodedPath() throws IOException { + void testRendererEncodedPath() throws Exception { CPDReportRenderer renderer = new XMLRenderer(); CpdReportBuilder builder = new CpdReportBuilder(); final String escapeChar = "&"; @@ -175,6 +202,7 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(builder.build(), sw); String report = sw.toString(); + assertReportIsValidSchema(report); assertThat(report, containsString(escapeChar)); } @@ -194,6 +222,7 @@ class XMLRendererTest { final StringWriter writer = new StringWriter(); renderer.render(report, writer); final String xmlOutput = writer.toString(); + assertReportIsValidSchema(xmlOutput); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(xmlOutput.getBytes(ENCODING))); final NodeList files = doc.getElementsByTagName("file"); @@ -219,6 +248,7 @@ class XMLRendererTest { final StringWriter writer = new StringWriter(); renderer.render(report, writer); final String xmlOutput = writer.toString(); + assertReportIsValidSchema(xmlOutput); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(new ByteArrayInputStream(xmlOutput.getBytes(ENCODING))); final NodeList files = doc.getElementsByTagName("file"); @@ -234,7 +264,7 @@ class XMLRendererTest { } @Test - void testRendererXMLEscaping() throws IOException { + void testRendererXMLEscaping() throws Exception { String codefragment = "code fragment" + FORM_FEED + "\nline2\nline3\nno & escaping necessary in CDATA\nx=\"]]>\";"; CPDReportRenderer renderer = new XMLRenderer(); @@ -251,10 +281,61 @@ class XMLRendererTest { StringWriter sw = new StringWriter(); renderer.render(builder.build(), sw); String report = sw.toString(); + assertReportIsValidSchema(report); assertThat(report, not(containsString(FORM_FEED))); assertThat(report, not(containsString(FORM_FEED_ENTITY))); assertThat(report, containsString("no & escaping necessary in CDATA")); assertThat(report, containsString("x=\"]]]]>\";")); assertThat(report, not(containsString("x=\"]]>\";"))); // must be escaped } + + @Test + void reportContainsProcessingError() throws Exception { + FileId fileId = FileId.fromPathLikeString("file1.txt"); + Report.ProcessingError processingError = new Report.ProcessingError( + new LexException(2, 1, fileId, "test exception", new RuntimeException("cause exception")), + fileId); + CPDReportRenderer renderer = new XMLRenderer(); + StringWriter sw = new StringWriter(); + renderer.render(CpdTestUtils.makeReport(Collections.emptyList(), Collections.emptyMap(), Collections.singletonList(processingError)), sw); + String report = sw.toString(); + assertReportIsValidSchema(report); + + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream(report.getBytes(ENCODING))); + NodeList nodes = doc.getChildNodes(); + Node n = nodes.item(0); + assertEquals("pmd-cpd", n.getNodeName()); + assertEquals(1, doc.getElementsByTagName("error").getLength()); + Node error = doc.getElementsByTagName("error").item(0); + String filename = error.getAttributes().getNamedItem("filename").getNodeValue(); + assertEquals(processingError.getFileId().getAbsolutePath(), filename); + String msg = error.getAttributes().getNamedItem("msg").getNodeValue(); + assertEquals(processingError.getMsg(), msg); + String textContent = error.getTextContent(); + assertEquals(processingError.getDetail(), textContent); + } + + private static void assertReportIsValidSchema(String report) throws SAXException, ParserConfigurationException, IOException { + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = schemaFactory.newSchema(new StreamSource(XMLRenderer.class.getResourceAsStream("/cpd-report_1_0_0.xsd"))); + + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setNamespaceAware(true); + saxParserFactory.setValidating(false); + saxParserFactory.setSchema(schema); + + SAXParser saxParser = saxParserFactory.newSAXParser(); + saxParser.parse(new InputSource(new StringReader(report)), new DefaultHandler() { + @Override + public void error(SAXParseException e) throws SAXException { + throw e; + } + + @Override + public void warning(SAXParseException e) throws SAXException { + throw e; + } + }); + } } diff --git a/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/SampleCpdReport.xml b/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/SampleCpdReport.xml index e0e6b0c465..ce6169ae59 100644 --- a/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/SampleCpdReport.xml +++ b/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/SampleCpdReport.xml @@ -1,8 +1,13 @@ - + - - + +