diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/security/xml/ApexCRUDViolation.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/security/xml/ApexCRUDViolation.xml
index 5d5234e28a..fbf529e30f 100644
--- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/security/xml/ApexCRUDViolation.xml
+++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/security/xml/ApexCRUDViolation.xml
@@ -1501,7 +1501,7 @@ public class Foo {
     </test-code>
 
 
-    <test-code regressionTest="false">
+    <test-code disabled="true">
         <!--
             Demonstrate that authorization for sub-relation queries doesn't work properly; see checkForAccessibility()
             See https://github.com/pmd/pmd/issues/2775
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/metrics/impl/xml/AtfdTest.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/metrics/impl/xml/AtfdTest.xml
index bf1a8554b5..ec603d1ec1 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/metrics/impl/xml/AtfdTest.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/metrics/impl/xml/AtfdTest.xml
@@ -146,7 +146,7 @@ public class StatementAndBraceFinder {
         <code-ref id="full-example"/>
     </test-code>
 
-    <test-code regressionTest="false">
+    <test-code disabled="true">
         <description>TODO: known limitation, should report 1</description>
         <!-- <rule-property name="reportClasses">false</rule-property> -->
         <expected-problems>1</expected-problems>
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedPrivateMethod.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedPrivateMethod.xml
index 219557b00f..2a7e09a935 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedPrivateMethod.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedPrivateMethod.xml
@@ -285,7 +285,7 @@ public class Foo {
         ]]></code>
     </test-code>
 
-    <test-code regressionTest="false">
+    <test-code disabled="true">
         <description>#46 False +: Unused private field: call to instance of self, received from another class</description>
         <expected-problems>0</expected-problems>
         <code><![CDATA[
diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/InvalidLogMessageFormat.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/InvalidLogMessageFormat.xml
index 253e972b8e..c06c27485a 100644
--- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/InvalidLogMessageFormat.xml
+++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/errorprone/xml/InvalidLogMessageFormat.xml
@@ -885,7 +885,7 @@ class InvalidLogMessageFormatTest {
         ]]></code>
     </test-code>
 
-    <test-code regressionTest="false">
+    <test-code disabled="true">
         <description>[java] InvalidLogMessageFormat false-negative for a lambda argument #2255</description>
         <expected-problems>3</expected-problems>
         <expected-linenumbers>11,13,16</expected-linenumbers>
diff --git a/pmd-modelica/src/test/resources/net/sourceforge/pmd/lang/modelica/rule/bestpractices/xml/AmbiguousResolution.xml b/pmd-modelica/src/test/resources/net/sourceforge/pmd/lang/modelica/rule/bestpractices/xml/AmbiguousResolution.xml
index b038ce6acd..b2fba5eb90 100644
--- a/pmd-modelica/src/test/resources/net/sourceforge/pmd/lang/modelica/rule/bestpractices/xml/AmbiguousResolution.xml
+++ b/pmd-modelica/src/test/resources/net/sourceforge/pmd/lang/modelica/rule/bestpractices/xml/AmbiguousResolution.xml
@@ -68,7 +68,7 @@ end Test;
         ]]></code>
     </test-code>
 
-    <test-code regressionTest="false">
+    <test-code disabled="true">
         <description>Invalid example: suspicious reference to X</description>
         <expected-problems>1</expected-problems>
         <code><![CDATA[
diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java
index 486a7ec718..b56bb03322 100644
--- a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java
+++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java
@@ -12,7 +12,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.w3c.dom.Attr;
@@ -240,7 +240,8 @@ class BaseTestParserImpl {
             }
             String propertyName = nameAttr.getNodeValue();
             if (knownProps.getPropertyDescriptor(propertyName) == null) {
-                Stream<String> knownNames = knownProps.getPropertyDescriptors().stream().map(PropertyDescriptor::name);
+                String knownNames = knownProps.getPropertyDescriptors().stream().map(PropertyDescriptor::name)
+                        .collect(Collectors.joining(", "));
                 err.at(nameAttr).error("Unknown property, known property names are {0}", knownNames);
                 continue;
             }
diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java
index 777d9e3560..56df86f143 100644
--- a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java
+++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/TestSchemaParser.java
@@ -19,6 +19,7 @@ import net.sourceforge.pmd.annotation.Experimental;
 import com.github.oowekyala.ooxml.messages.NiceXmlMessageSpec;
 import com.github.oowekyala.ooxml.messages.OoxmlFacade;
 import com.github.oowekyala.ooxml.messages.PositionedXmlDoc;
+import com.github.oowekyala.ooxml.messages.PrintStreamMessageHandler;
 import com.github.oowekyala.ooxml.messages.XmlException;
 import com.github.oowekyala.ooxml.messages.XmlMessageReporter;
 import com.github.oowekyala.ooxml.messages.XmlMessageReporterBase;
@@ -57,7 +58,9 @@ public class TestSchemaParser {
      * @throws XmlException If parsing throws this
      */
     public RuleTestCollection parse(Rule rule, InputSource inputSource) throws IOException, XmlException {
-        OoxmlFacade ooxml = new OoxmlFacade();
+        // note: need to explicitly specify the writer here, so that in unit tests
+        // System.err can be swapped out and in
+        OoxmlFacade ooxml = new OoxmlFacade().withPrinter(new PrintStreamMessageHandler(System.err));
         PositionedXmlDoc doc = ooxml.parse(newDocumentBuilder(), inputSource);
 
         try (PmdXmlReporterImpl err = new PmdXmlReporterImpl(ooxml, doc.getPositioner())) {
diff --git a/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java b/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java
index 7713d5a6fe..7b18e62500 100644
--- a/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java
+++ b/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java
@@ -6,6 +6,7 @@ package net.sourceforge.pmd.test.schema;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -86,6 +87,31 @@ public class TestSchemaParserTest {
                                                                           + "                   ^^^^^^^^^^^^^^ Attribute 'regressionTest' is deprecated, use 'ignored' with inverted value\n"));
     }
 
+    @Test
+    public void testUnknownProperty() throws IOException {
+        String file = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                + "<test-data\n"
+                + "        xmlns=\"http://pmd.sourceforge.net/rule-tests\"\n"
+                + "        xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n"
+                + "        xsi:schemaLocation=\"http://pmd.sourceforge.net/rule-tests net/sourceforge/pmd/test/schema/rule-tests_1_0_0.xsd\">\n"
+                + "    <test-code>\n"
+                + "        <description>equality operators with Double.NaN</description>\n"
+                + "        <rule-property name='invalid_property'>foo</rule-property>\n"
+                + "        <expected-problems>0</expected-problems>\n"
+                + "        <code><![CDATA[\n"
+                + "            public class Foo {\n"
+                + "            }\n"
+                + "            ]]></code>\n"
+                + "    </test-code>\n"
+                + "</test-data>\n";
+
+        errStreamCaptor.enableLog();
+        assertThrows(IllegalStateException.class, () -> parseFile(file));
+
+        MatcherAssert.assertThat(errStreamCaptor.getLog(), containsString("  8|         <rule-property name='invalid_property'>foo</rule-property>\n"
+              + "                            ^^^^ Unknown property, known property names are violationSuppressRegex, violationSuppressXPath, testIntProperty\n"));
+    }
+
     private RuleTestCollection parseFile(String file) throws IOException {
         MockRule mockRule = new MockRule();
         mockRule.setLanguage(PlainTextLanguage.getInstance());