diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/RuleSetSchemaTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/RuleSetSchemaTest.java
index 69cb63ed81..5e63450470 100644
--- a/pmd-core/src/test/java/net/sourceforge/pmd/RuleSetSchemaTest.java
+++ b/pmd-core/src/test/java/net/sourceforge/pmd/RuleSetSchemaTest.java
@@ -5,11 +5,13 @@
 package net.sourceforge.pmd;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -19,14 +21,18 @@ import javax.xml.XMLConstants;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.stream.StreamSource;
 import javax.xml.validation.Schema;
 import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
 
 import org.junit.Before;
 import org.junit.Test;
 import org.w3c.dom.Attr;
 import org.w3c.dom.Document;
+import org.xml.sax.EntityResolver;
 import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
@@ -35,7 +41,7 @@ public class RuleSetSchemaTest {
     private CollectingErrorHandler errorHandler;
 
     @Before
-    public void setup() {
+    public void setUp() {
         Locale.setDefault(Locale.ROOT);
         errorHandler = new CollectingErrorHandler();
     }
@@ -63,45 +69,63 @@ public class RuleSetSchemaTest {
         assertEquals("true", ((Attr) doc.getElementsByTagName("rule").item(0).getAttributes().getNamedItem("metrics")).getValue());
     }
 
+    @Test
+    public void validateOnly() throws Exception {
+        Validator validator = PMDRuleSetEntityResolver.getSchemaVersion2().newValidator();
+        validator.setErrorHandler(errorHandler);
+        validator.validate(new StreamSource(new ByteArrayInputStream(generateRuleSet("2.0.0", false).getBytes(StandardCharsets.UTF_8))));
+        assertTrue(errorHandler.isValid());
+        errorHandler.reset();
+
+        validator.validate(new StreamSource(new ByteArrayInputStream(generateRuleSet("2.0.0", true).getBytes(StandardCharsets.UTF_8))));
+        assertFalse(errorHandler.isValid()); // metrics attribute is not allowed
+        errorHandler.reset();
+
+        validator.validate(new StreamSource(new ByteArrayInputStream(generateRuleSet("3.0.0", false).getBytes(StandardCharsets.UTF_8))));
+        assertFalse(errorHandler.isValid()); // schema namespace doesn't match, so element ruleset is not known
+        errorHandler.reset();
+
+        validator = PMDRuleSetEntityResolver.getSchemaVersion3().newValidator();
+        validator.setErrorHandler(errorHandler);
+        validator.validate(new StreamSource(new ByteArrayInputStream(generateRuleSet("3.0.0", false).getBytes(StandardCharsets.UTF_8))));
+        assertTrue(errorHandler.isValid());
+        errorHandler.reset();
+
+        validator.validate(new StreamSource(new ByteArrayInputStream(generateRuleSet("3.0.0", true).getBytes(StandardCharsets.UTF_8))));
+        assertTrue(errorHandler.isValid());
+        errorHandler.reset();
+    }
+
     private Document parseWithVersion2(String ruleset) throws SAXException, ParserConfigurationException, IOException {
         DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
         dbf.setNamespaceAware(true);
-        dbf.setSchema(loadSchemaVersion2());
+        dbf.setFeature("http://apache.org/xml/features/validation/schema", true);
         DocumentBuilder builder = dbf.newDocumentBuilder();
         builder.setErrorHandler(errorHandler);
+        builder.setEntityResolver(new PMDRuleSetEntityResolver());
+
         Document doc = builder.parse(new ByteArrayInputStream(ruleset.getBytes(StandardCharsets.UTF_8)));
         return doc;
     }
 
-    private Schema loadSchemaVersion2() throws SAXException {
-        Schema schema = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
-                .newSchema(RuleSetFactory.class.getResource("/ruleset_2_0_0.xsd"));
-        return schema;
-    }
-
     private Document parseWithVersion3(String ruleset) throws SAXException, ParserConfigurationException, IOException {
         DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
         dbf.setNamespaceAware(true);
-        dbf.setSchema(loadSchemaVersion3());
+        dbf.setFeature("http://apache.org/xml/features/validation/schema", true);
         DocumentBuilder builder = dbf.newDocumentBuilder();
         builder.setErrorHandler(errorHandler);
         Document doc = builder.parse(new ByteArrayInputStream(ruleset.getBytes(StandardCharsets.UTF_8)));
         return doc;
     }
 
-    private Schema loadSchemaVersion3() throws SAXException {
-        Schema schema = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
-                .newSchema(RuleSetFactory.class.getResource("/ruleset_3_0_0.xsd"));
-        return schema;
-    }
-
     private String generateRuleSet(String version, boolean withMetrics) {
         String versionUnderscore = version.replaceAll("\\.", "_");
         String ruleset = "<?xml version=\"1.0\"?>" + PMD.EOL
-                + "<ruleset name=\"Custom ruleset\"" + PMD.EOL
+                + "<ruleset " + PMD.EOL
                 + "    xmlns=\"http://pmd.sourceforge.net/ruleset/" + version + "\"" + PMD.EOL
                 + "    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" + PMD.EOL
-                + "    xsi:schemaLocation=\"http://pmd.sourceforge.net/ruleset/" + version + " http://pmd.sourceforge.net/ruleset_" + versionUnderscore + ".xsd\">" + PMD.EOL
+                + "    xsi:schemaLocation=\"http://pmd.sourceforge.net/ruleset/" + version + " http://pmd.sourceforge.net/ruleset_" + versionUnderscore + ".xsd\""
+                + "    name=\"Custom ruleset\" >" + PMD.EOL
                 + "  <description>" + PMD.EOL
                 + "  This ruleset checks my code for bad stuff" + PMD.EOL
                 + "  </description>" + PMD.EOL
@@ -124,6 +148,30 @@ public class RuleSetSchemaTest {
         return ruleset;
     }
 
+    public static class PMDRuleSetEntityResolver implements EntityResolver {
+        private static URL schema2 = RuleSetFactory.class.getResource("/ruleset_2_0_0.xsd");
+        private static URL schema3 = RuleSetFactory.class.getResource("/ruleset_3_0_0.xsd");
+        private static SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+
+        @Override
+        public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
+            if ("http://pmd.sourceforge.net/ruleset_2_0_0.xsd".equals(systemId)) {
+                return new InputSource(schema2.toExternalForm());
+            } else if ("http://pmd.sourceforge.net/ruleset_3_0_0.xsd".equals(systemId)) {
+                return new InputSource(schema3.toExternalForm());
+            }
+            throw new IllegalArgumentException("Unable to resolve entity (publicId=" + publicId + ", systemId=" + systemId + ")");
+        }
+
+        public static Schema getSchemaVersion2() throws SAXException {
+            return schemaFactory.newSchema(schema2);
+        }
+
+        public static Schema getSchemaVersion3() throws SAXException {
+            return schemaFactory.newSchema(schema3);
+        }
+    }
+
     public static class CollectingErrorHandler implements ErrorHandler {
         private List<SAXParseException> warnings = new ArrayList<>();
         private List<SAXParseException> errors = new ArrayList<>();
@@ -164,5 +212,11 @@ public class RuleSetSchemaTest {
         public String toString() {
             return "Warnings: " + warnings + "; Errors: " + errors + "; Fatal Errors: " + fatalErrors;
         }
+
+        public void reset() {
+            warnings.clear();
+            errors.clear();
+            fatalErrors.clear();
+        }
     }
 }