diff --git a/pmd-javascript/src/test/java/net/sourceforge/pmd/cli/CLITest.java b/pmd-javascript/src/test/java/net/sourceforge/pmd/cli/CLITest.java
index 3b83aaa055..ba086299f0 100644
--- a/pmd-javascript/src/test/java/net/sourceforge/pmd/cli/CLITest.java
+++ b/pmd-javascript/src/test/java/net/sourceforge/pmd/cli/CLITest.java
@@ -8,6 +8,8 @@ import static org.hamcrest.MatcherAssert.assertThat;
 
 import org.junit.Test;
 
+import net.sourceforge.pmd.PMD.StatusCode;
+
 /**
  * @author Romain Pelisse <belaran@gmail.com>
  *
@@ -15,9 +17,18 @@ import org.junit.Test;
 public class CLITest extends BaseCLITest {
     @Test
     public void useEcmaScript() {
-        String[] args = { "-d", SOURCE_FOLDER, "-f", "xml", "-R", "ecmascript-basic", "-version", "3", "-l",
-            "ecmascript", "-debug", };
-        String log = runTest(args);
+        String log = runTest(StatusCode.VIOLATIONS_FOUND,
+                             "-d",
+                             SOURCE_FOLDER,
+                             "-f",
+                             "xml",
+                             "-R",
+                             "rulesets/testing/js-rset1.xml",
+                             "-version",
+                             "3",
+                             "-l",
+                             "ecmascript",
+                             "--debug");
         assertThat(log, containsPattern("Adding file .*\\.js \\(lang: ecmascript 3\\)"));
     }
 }
diff --git a/pmd-javascript/src/test/java/net/sourceforge/pmd/lang/ecmascript/DummyJsRule.java b/pmd-javascript/src/test/java/net/sourceforge/pmd/lang/ecmascript/DummyJsRule.java
new file mode 100644
index 0000000000..50a5690238
--- /dev/null
+++ b/pmd-javascript/src/test/java/net/sourceforge/pmd/lang/ecmascript/DummyJsRule.java
@@ -0,0 +1,37 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.ecmascript;
+
+import java.util.List;
+
+import net.sourceforge.pmd.RuleContext;
+import net.sourceforge.pmd.lang.ast.Node;
+import net.sourceforge.pmd.lang.ecmascript.rule.AbstractEcmascriptRule;
+
+/**
+ * @author Clément Fournier
+ */
+public class DummyJsRule extends AbstractEcmascriptRule {
+
+    @Override
+    public void apply(List<? extends Node> nodes, RuleContext ctx) {
+        for (Node node : nodes) {
+            apply(node, ctx);
+        }
+    }
+
+    public void apply(Node node, RuleContext ctx) {
+
+    }
+
+    public static class DummyRuleOneViolationPerFile extends DummyJsRule {
+
+        @Override
+        public void apply(Node node, RuleContext ctx) {
+            ctx.addViolation(node);
+        }
+    }
+
+}
diff --git a/pmd-javascript/src/test/resources/rulesets/testing/js-rset1.xml b/pmd-javascript/src/test/resources/rulesets/testing/js-rset1.xml
new file mode 100644
index 0000000000..b934c00b24
--- /dev/null
+++ b/pmd-javascript/src/test/resources/rulesets/testing/js-rset1.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<ruleset name="Test Ruleset" xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
+
+    <description>
+        Ruleset used by test RuleSetReferenceIdTest
+    </description>
+
+    <rule name="DummyRuleWithAViolationPerFile"
+          language="ecmascript"
+          since="1.0"
+          message="Violation from test-rset-1.xml"
+          class="net.sourceforge.pmd.lang.ecmascript.DummyJsRule$DummyRuleOneViolationPerFile">
+        <description>
+            Just for test
+        </description>
+        <priority>3</priority>
+        <example>
+ <![CDATA[
+            ]]>
+     </example>
+    </rule>
+
+</ruleset>
diff --git a/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCLITest.java b/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCLITest.java
index 54e9739159..8756256260 100644
--- a/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCLITest.java
+++ b/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCLITest.java
@@ -15,13 +15,14 @@ import java.io.PrintStream;
 import java.nio.file.Files;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.IOUtils;
+import org.apache.tools.ant.util.TeeOutputStream;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.contrib.java.lang.system.SystemErrRule;
+import org.junit.contrib.java.lang.system.SystemOutRule;
 
 import net.sourceforge.pmd.PMD;
 import net.sourceforge.pmd.PMD.StatusCode;
@@ -39,8 +40,10 @@ public abstract class BaseCLITest {
     // and slowing down tests
     protected static final String SOURCE_FOLDER = "src/test/resources/net/sourceforge/pmd/cli";
 
-    protected PrintStream originalOut;
-    protected PrintStream originalErr;
+    @Rule
+    public SystemErrRule systemErrRule = new SystemErrRule().muteForSuccessfulTests();
+    @Rule
+    public SystemOutRule systemOutRule = new SystemOutRule().muteForSuccessfulTests();
 
     /**
      * @throws java.lang.Exception
@@ -55,20 +58,6 @@ public abstract class BaseCLITest {
         }
     }
 
-    @Before
-    public void setup() {
-        originalOut = System.out;
-        originalErr = System.err;
-    }
-
-    @After
-    public void tearDown() {
-        IOUtils.closeQuietly(System.out);
-
-        System.setOut(originalOut);
-        System.setErr(originalErr);
-    }
-
     protected void createTestOutputFile(String filename) {
         try {
             @SuppressWarnings("PMD.CloseResource")
@@ -133,9 +122,11 @@ public abstract class BaseCLITest {
 
     protected String runTest(StatusCode expectedExitCode, String... args) {
         ByteArrayOutputStream console = new ByteArrayOutputStream();
-        PrintStream out = new PrintStream(console);
+
+        PrintStream out = new PrintStream(new TeeOutputStream(console, System.out));
+        PrintStream err = new PrintStream(new TeeOutputStream(console, System.err));
         System.setOut(out);
-        System.setErr(out);
+        System.setErr(err);
         StatusCode statusCode = PMD.runPmd(args);
         assertEquals(expectedExitCode, statusCode);
         return console.toString();