From 60139c8f4a66f4ee474ed02adf8f3cebf9beb27a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Tue, 17 May 2022 23:32:57 +0200 Subject: [PATCH] Most of the work to extract module Need a release of nice-xml-messages Need an update to pmd-test Need to add the 'ignored' attribute to schema --- pmd-test-schema/pom.xml | 39 +++ .../pmd/test/schema/BaseTestParserImpl.java | 277 ++++++++++++++++++ .../pmd/test/schema/TestCollection.java | 22 ++ .../pmd/test/schema/TestDescriptor.java | 79 +++++ .../pmd/test/schema/TestSchemaParser.java | 163 +++++++++++ .../pmd/test/schema/TestSchemaVersion.java | 53 ++++ .../pmd/test/schema/package-info.java | 5 + .../pmd/test.schema/rule-tests_1_0_0.xsd | 64 ++++ pmd-test/pom.xml | 4 + pom.xml | 1 + 10 files changed, 707 insertions(+) create mode 100644 pmd-test-schema/pom.xml create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestCollection.java create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestDescriptor.java create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaVersion.java create mode 100644 pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/package-info.java create mode 100644 pmd-test-schema/src/main/resources/net/sourceforge/pmd/test.schema/rule-tests_1_0_0.xsd diff --git a/pmd-test-schema/pom.xml b/pmd-test-schema/pom.xml new file mode 100644 index 0000000000..e6761f477d --- /dev/null +++ b/pmd-test-schema/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + pmd-test-schema + PMD Test Schema + + Parser for the XML test description format. The module has no dependency + on junit or other test-only dependencies. + + + + pmd + net.sourceforge.pmd + 6.46.0-SNAPSHOT + ../ + + + + 8 + + + + + + net.sourceforge.pmd + pmd-core + + + com.github.oowekyala.ooxml + nice-xml-messages + 2.0 + + + + + + diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java new file mode 100644 index 0000000000..9e60cb5bfc --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java @@ -0,0 +1,277 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.test.schema; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Stream; + +import org.w3c.dom.Attr; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import net.sourceforge.pmd.Rule; +import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.properties.PropertyDescriptor; +import net.sourceforge.pmd.properties.PropertySource; +import net.sourceforge.pmd.test.schema.TestSchemaParser.PmdXmlReporter; + +import com.github.oowekyala.ooxml.DomUtils; + +/** + * @author Clément Fournier + */ +class BaseTestParserImpl { + + static class ParserV1 extends BaseTestParserImpl { + + } + + public TestCollection parseDocument(Rule rule, Document doc, PmdXmlReporter err) { + Element root = doc.getDocumentElement(); + + Map codeFragments = parseCodeFragments(err, root); + + Set usedFragments = new HashSet<>(); + NodeList testCodes = root.getElementsByTagName("test-code"); + TestCollection result = new TestCollection(); + for (int i = 0; i < testCodes.getLength(); i++) { + TestDescriptor descriptor = new TestDescriptor(i, rule.deepCopy()); + + PmdXmlReporter errScope = err.newScope(); + parseSingleTest((Element) testCodes.item(i), descriptor, codeFragments, usedFragments, errScope); + if (!errScope.hasError()) { + result.addTest(descriptor); + } + } + + codeFragments.keySet().removeAll(usedFragments); + codeFragments.forEach((id, node) -> err.at(node).warn("Unused code fragment")); + + return result; + } + + private Map parseCodeFragments(PmdXmlReporter err, Element root) { + Map codeFragments = new HashMap<>(); + + for (Node node : DomUtils.asList(root.getElementsByTagName("code-fragment"))) { + Attr id = getRequiredAttribute("id", (Element) node, err); + if (id == null) { + continue; + } + + Element prev = codeFragments.put(id.getValue(), (Element) node); + if (prev != null) { + err.at(prev).error("Fragment with duplicate id ''{0}'' is ignored", id.getValue()); + } + } + return codeFragments; + } + + private void parseSingleTest(Element testCode, TestDescriptor descriptor, Map fragments, Set usedFragments, PmdXmlReporter err) { + { + String description = getSingleChildText(testCode, "description", true, err); + if (description == null) { + return; + } + descriptor.setDescription(description); + } + + parseBoolAttribute(testCode, "reinitializeRule", true, err, "Attribute 'reinitializeRule' is deprecated and ignored, assumed true"); + parseBoolAttribute(testCode, "useAuxClasspath", true, err, "Attribute 'useAuxClasspath' is deprecated and ignored, assumed true"); + + boolean ignored = parseBoolAttribute(testCode, "ignored", false, err, null) + && !parseBoolAttribute(testCode, "regressionTest", true, err, "Attribute ''regressionTest'' is deprecated, use ''ignored'' with inverted value"); + + descriptor.setIgnored(ignored); + + Properties properties = parseRuleProperties(testCode, descriptor.getRule(), err); + descriptor.getProperties().putAll(properties); + + parseExpectedProblems(testCode, descriptor, err); + + String code = getTestCode(testCode, fragments, err); + if (code == null) { + return; + } + descriptor.setCode(code); + + + LanguageVersion lversion = parseLanguageVersion(testCode, err); + descriptor.setLanguageVersion(lversion); + } + + private void parseExpectedProblems(Element testCode, TestDescriptor descriptor, PmdXmlReporter err) { + Node expectedProblemsNode = getSingleChild(testCode, "expected-problems", true, err); + if (expectedProblemsNode == null) { + return; + } + int expectedProblems = Integer.parseInt(parseTextNode(expectedProblemsNode)); + + List expectedMessages = null; + { + Element messagesNode = getSingleChild(testCode, "expected-messages", false, err); + if (messagesNode != null) { + expectedMessages = new ArrayList<>(); + List messageNodes = DomUtils.asList(messagesNode.getElementsByTagName("message")); + if (messageNodes.size() != expectedProblems) { + err.at(expectedProblemsNode).error("Number of ''expected-messages'' ({0}) does not match", messageNodes.size()); + return; + } + + for (Node message : messageNodes) { + expectedMessages.add(parseTextNode(message)); + } + } + } + + List expectedLineNumbers = null; + { + Element lineNumbers = getSingleChild(testCode, "expected-linenumbers", false, err); + if (lineNumbers != null) { + expectedLineNumbers = new ArrayList<>(); + String[] linenos = parseTextNode(lineNumbers).split(","); + if (linenos.length != expectedProblems) { + err.at(expectedProblemsNode).error("Number of ''expected-linenumbers'' ({0}) does not match", linenos.length); + return; + } + for (String num : linenos) { + expectedLineNumbers.add(Integer.valueOf(num.trim())); + } + } + } + + descriptor.recordExpectedViolations( + expectedProblems, + expectedLineNumbers, + expectedMessages + ); + + } + + private String getTestCode(Element testCode, Map fragments, PmdXmlReporter err) { + String code = getSingleChildText(testCode, "code", false, err); + if (code == null) { + // Should have a coderef + NodeList coderefs = testCode.getElementsByTagName("code-ref"); + if (coderefs.getLength() == 0) { + throw new RuntimeException( + "Required tag is missing from the test-xml. Supply either a code or a code-ref tag"); + } + Element coderef = (Element) coderefs.item(0); + Attr id = getRequiredAttribute("id", coderef, err); + if (id == null) { + return null; + } + Element fragment = fragments.get(id.getValue()); + if (fragment == null) { + err.at(id).error("Unknown id, known IDs are {0}", fragments.keySet()); + return null; + } + code = parseTextNodeNoTrim(fragment); + } + return code; + } + + private LanguageVersion parseLanguageVersion(Element testCode, PmdXmlReporter err) { + Node sourceTypeNode = getSingleChild(testCode, "source-type", false, err); + if (sourceTypeNode == null) { + return null; + } + String languageVersionString = parseTextNode(sourceTypeNode); + LanguageVersion languageVersion = LanguageRegistry.findLanguageVersionByTerseName(languageVersionString); + if (languageVersion != null) { + return languageVersion; + } + + err.at(sourceTypeNode).error("Unknown language version ''{0}''", languageVersionString); + return null; + } + + private Properties parseRuleProperties(Element testCode, PropertySource knownProps, PmdXmlReporter err) { + Properties properties = new Properties(); + for (Node ruleProperty : DomUtils.asList(testCode.getElementsByTagName("rule-property"))) { + Node nameAttr = getRequiredAttribute("name", (Element) ruleProperty, err); + if (nameAttr == null) { + continue; + } + String propertyName = nameAttr.getNodeValue(); + if (knownProps.getPropertyDescriptor(propertyName) == null) { + Stream knownNames = knownProps.getPropertyDescriptors().stream().map(PropertyDescriptor::name); + err.at(nameAttr).error("Unknown property, known property names are {0}", knownNames); + continue; + } + properties.setProperty(propertyName, parseTextNode(ruleProperty)); + } + return properties; + } + + private Attr getRequiredAttribute(String name, Element ruleProperty, PmdXmlReporter err) { + Attr nameAttr = (Attr) ruleProperty.getAttributes().getNamedItem(name); + if (nameAttr == null) { + err.at(ruleProperty).error("Missing ''{0}'' attribute", name); + return null; + } + return nameAttr; + } + + private boolean parseBoolAttribute(Element testCode, String attrName, boolean defaultValue, PmdXmlReporter err, String deprecationMessage) { + Attr attrNode = testCode.getAttributeNode(attrName); + if (attrNode != null) { + if (deprecationMessage != null) { + err.at(attrNode).warn(deprecationMessage); + } + return Boolean.parseBoolean(attrNode.getNodeValue()); + } + return defaultValue; + } + + + private String getSingleChildText(Element parentElm, String nodeName, boolean required, PmdXmlReporter err) { + Node node = getSingleChild(parentElm, nodeName, required, err); + if (node == null) { + return null; + } + return parseTextNode(node); + } + + private Element getSingleChild(Element parentElm, String nodeName, boolean required, PmdXmlReporter err) { + NodeList nodes = parentElm.getElementsByTagName(nodeName); + if (nodes.getLength() == 0) { + if (required) { + err.at(parentElm).error("Required child ''{0}'' is missing", nodeName); + } + return null; + } else if (nodes.getLength() > 1) { + err.at(nodes.item(1)).error("Duplicate tag ''{0}'' is ignored", nodeName); + } + return (Element) nodes.item(0); + } + + private static String parseTextNode(Node exampleNode) { + return parseTextNodeNoTrim(exampleNode).trim(); + } + + private static String parseTextNodeNoTrim(Node exampleNode) { + StringBuilder buffer = new StringBuilder(); + for (int i = 0; i < exampleNode.getChildNodes().getLength(); i++) { + Node node = exampleNode.getChildNodes().item(i); + if (node.getNodeType() == Node.CDATA_SECTION_NODE || node.getNodeType() == Node.TEXT_NODE) { + buffer.append(node.getNodeValue()); + } + } + return buffer.toString(); + } + + +} diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestCollection.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestCollection.java new file mode 100644 index 0000000000..c9e189144b --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestCollection.java @@ -0,0 +1,22 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.test.schema; + +import java.util.List; +import java.util.Objects; + +/** + * @author Clément Fournier + */ +public class TestCollection { + + private List tests; + + + public void addTest(TestDescriptor descriptor) { + tests.add(Objects.requireNonNull(descriptor)); + } + +} diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestDescriptor.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestDescriptor.java new file mode 100644 index 0000000000..b68814e382 --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestDescriptor.java @@ -0,0 +1,79 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.test.schema; + +import java.util.List; +import java.util.Properties; + +import net.sourceforge.pmd.Rule; +import net.sourceforge.pmd.lang.LanguageVersion; + +/** + * @author Clément Fournier + */ +public class TestDescriptor { + + private boolean ignored; + private String description; + private LanguageVersion languageVersion; + private Properties properties = new Properties(); + private int index; + private final Rule rule; + private String code; + private int expectedProblems; + private List expectedLineNumbers; + private List expectedMessages; + + public TestDescriptor(int index, Rule rule) { + this.index = index; + this.rule = rule; + } + + public Rule getRule() { + return rule; + } + + public Properties getProperties() { + return properties; + } + + public boolean isIgnored() { + return ignored; + } + + public void setIgnored(boolean ignored) { + this.ignored = ignored; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public LanguageVersion getLanguageVersion() { + return languageVersion; + } + + public void setLanguageVersion(LanguageVersion languageVersion) { + this.languageVersion = languageVersion; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public void recordExpectedViolations(int expectedProblems, List expectedLineNumbers, List expectedMessages) { + this.expectedProblems = expectedProblems; + this.expectedLineNumbers = expectedLineNumbers; + this.expectedMessages = expectedMessages; + } +} diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java new file mode 100644 index 0000000000..a53fb35fec --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java @@ -0,0 +1,163 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.test.schema; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.function.Consumer; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.xml.sax.InputSource; + +import net.sourceforge.pmd.Rule; +import net.sourceforge.pmd.annotation.Experimental; + +import com.github.oowekyala.ooxml.messages.NiceXmlMessageSpec; +import com.github.oowekyala.ooxml.messages.OoxmlFacade; +import com.github.oowekyala.ooxml.messages.PositionedXmlDoc; +import com.github.oowekyala.ooxml.messages.XmlException; +import com.github.oowekyala.ooxml.messages.XmlMessageReporter; +import com.github.oowekyala.ooxml.messages.XmlMessageReporterBase; +import com.github.oowekyala.ooxml.messages.XmlPosition; +import com.github.oowekyala.ooxml.messages.XmlPositioner; +import com.github.oowekyala.ooxml.messages.XmlSeverity; + + +/** + * Entry point to parse a test file. + * + * @author Clément Fournier + */ +@Experimental +public class TestSchemaParser { + + private final TestSchemaVersion version; + + public TestSchemaParser(TestSchemaVersion version) { + this.version = version; + } + + /** + * Entry point to parse a test file. + * + * @param rule Rule which owns the tests + * @param inputSource Where to access the test file to parse + * + * @return A test collection, possibly incomplete + * + * @throws IOException If parsing throws this + * @throws XmlException If parsing throws this + */ + public TestCollection parse(Rule rule, InputSource inputSource) throws IOException, XmlException { + OoxmlFacade ooxml = new OoxmlFacade(); + PositionedXmlDoc doc = ooxml.parse(newDocumentBuilder(), inputSource); + + try (PmdXmlReporterImpl err = new PmdXmlReporterImpl(ooxml, doc.getPositioner())) { + TestCollection collection = version.getParserImpl().parseDocument(rule, doc.getDocument(), err); + if (err.hasError()) { + // todo maybe add a way not to throw here + throw new IllegalStateException("Errors were encountered while parsing XML tests"); + } + return collection; + } + } + + interface PmdXmlReporter extends XmlMessageReporter { + + boolean hasError(); + + PmdXmlReporter newScope(); + } + + private static class PmdXmlReporterImpl + extends XmlMessageReporterBase + implements PmdXmlReporter { + + private boolean hasError; + + protected PmdXmlReporterImpl(OoxmlFacade ooxml, + XmlPositioner positioner) { + super(ooxml, positioner); + } + + @Override + protected Reporter create2ndStage(XmlPosition position, XmlPositioner positioner, Consumer handleEx) { + return new Reporter(position, positioner, ooxml, handleEx.andThen(this::consumeEx)); + } + + protected void consumeEx(XmlException e) { + hasError |= e.getSeverity() == XmlSeverity.ERROR; + } + + @Override + public PmdXmlReporter newScope() { + return new PmdXmlReporterImpl(ooxml, positioner) { + @Override + protected void consumeEx(XmlException e) { + super.consumeEx(e); + PmdXmlReporterImpl.this.consumeEx(e); + } + }; + } + + @Override + public boolean hasError() { + return hasError; + } + + @Override + public void close() { + } + } + + private DocumentBuilder newDocumentBuilder() { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + try { + dbf.setSchema(version.getSchema()); + dbf.setNamespaceAware(true); + return dbf.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } + } + + + static class Reporter { + + private final XmlPosition position; + private final XmlPositioner positioner; + private final OoxmlFacade ooxml; + + private final Consumer handler; + + private Reporter(XmlPosition position, XmlPositioner positioner, OoxmlFacade ooxml, Consumer handler) { + this.position = position; + this.positioner = positioner; + this.ooxml = ooxml; + this.handler = handler; + } + + public void warn(String messageFormat, Object... args) { + reportImpl(XmlSeverity.WARNING, MessageFormat.format(messageFormat, args)); + + } + + public void error(String messageFormat, Object... args) { + reportImpl(XmlSeverity.ERROR, MessageFormat.format(messageFormat, args)); + } + + private void reportImpl(XmlSeverity severity, String formattedMessage) { + NiceXmlMessageSpec spec = + new NiceXmlMessageSpec(position, formattedMessage) + .withSeverity(severity); + String fullMessage = ooxml.getFormatter().formatSpec(ooxml, spec, positioner); + XmlException ex = new XmlException(spec, fullMessage); + handler.accept(ex); + } + } + +} diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaVersion.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaVersion.java new file mode 100644 index 0000000000..933bd9d758 --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaVersion.java @@ -0,0 +1,53 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.test.schema; + +import java.net.URL; +import java.util.Objects; +import javax.xml.XMLConstants; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; + +import org.xml.sax.SAXException; + +/** + * @author Clément Fournier + */ +public enum TestSchemaVersion { + V1("rule-tests_1_0_0.xsd", new BaseTestParserImpl.ParserV1()); + + private final Schema schema; + private String schemaLoc; + private BaseTestParserImpl parser; + + TestSchemaVersion(String schemaLoc, BaseTestParserImpl parser) { + this.schemaLoc = schemaLoc; + this.parser = parser; + this.schema = parseSchema(); + } + + BaseTestParserImpl getParserImpl() { + return parser; + } + + public Schema getSchema() { + return schema; + } + + private Schema parseSchema() { + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + try { + return schemaFactory.newSchema(locateSchema()); + } catch (SAXException e) { + throw new RuntimeException("Cannot parse schema " + this, e); + } + } + + private URL locateSchema() { + URL resource = TestSchemaVersion.class.getResource(schemaLoc); + return Objects.requireNonNull(resource, "Cannot find schema location " + schemaLoc); + } + +} diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/package-info.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/package-info.java new file mode 100644 index 0000000000..467c3649dd --- /dev/null +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/package-info.java @@ -0,0 +1,5 @@ +/** + * Contains a parser for the XML test format. + */ + +package net.sourceforge.pmd.test.schema; diff --git a/pmd-test-schema/src/main/resources/net/sourceforge/pmd/test.schema/rule-tests_1_0_0.xsd b/pmd-test-schema/src/main/resources/net/sourceforge/pmd/test.schema/rule-tests_1_0_0.xsd new file mode 100644 index 0000000000..e1fcd18fde --- /dev/null +++ b/pmd-test-schema/src/main/resources/net/sourceforge/pmd/test.schema/rule-tests_1_0_0.xsd @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pmd-test/pom.xml b/pmd-test/pom.xml index 6922c448a0..9fc7654eef 100644 --- a/pmd-test/pom.xml +++ b/pmd-test/pom.xml @@ -12,6 +12,10 @@ ../ + + 8 + +