[core] Fix XMLRenderer with UTF-16

When using UTF-16 as encoding, the XMLRenderer produced
invalid XML: When inserting the linebreak encoded with the
given encoding "UTF-16", a BOM was created. This inserted additional
characters U+FEFF in the middle of the file, which is not allowed.

A partial workaround for this issue would be, to use "UTF-16BE" as
encoding instead. This doesn't create a BOM. However, the resulting XML
file is then completely without a BOM.

pmd-core/src/main/java/net/sourceforge/pmd/renderers/XMLRenderer.java

@@ -10,6 +10,8 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.io.Writer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -66,7 +68,8 @@ public class XMLRenderer extends AbstractIncrementingRenderer {
     @Override
     public void start() throws IOException {
         String encoding = getProperty(ENCODING);
-        lineSeparator = PMD.EOL.getBytes(encoding);
+        String unmarkedEncoding = toUnmarkedEncoding(encoding);
+        lineSeparator = PMD.EOL.getBytes(unmarkedEncoding);
 
         try {
             xmlWriter.writeStartDocument(encoding, "1.0");
@@ -86,6 +89,26 @@ public class XMLRenderer extends AbstractIncrementingRenderer {
         }
     }
 
+    /**
+     * Return a encoding, which doesn't write a BOM (byte order mark).
+     * Only UTF-16 encoders might write a BOM, see {@link Charset}.
+     *
+     * <p>This is needed, so that we don't accidentally add BOMs whenever
+     * we insert a newline.
+     *
+     * @return
+     */
+    private static String toUnmarkedEncoding(String encoding) {
+        if (StandardCharsets.UTF_16.name().equalsIgnoreCase(encoding)) {
+            return StandardCharsets.UTF_16BE.name();
+        }
+        // edge case: UTF-16LE with BOM
+        if ("UTF-16LE_BOM".equalsIgnoreCase(encoding)) {
+            return StandardCharsets.UTF_16LE.name();
+        }
+        return encoding;
+    }
+
     /**
      * Outputs a platform dependent line separator.
      *

pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java

@@ -123,6 +123,12 @@ public class XMLRendererTest extends AbstractRendererTest {
         verifyXmlEscaping(renderer, "\ud801\udc1c", StandardCharsets.UTF_8);
     }
 
+    @Test
+    public void testXMLEscapingWithUTF16() throws Exception {
+        Renderer renderer = getRenderer();
+        verifyXmlEscaping(renderer, "𐐜", StandardCharsets.UTF_16);
+    }
+
     @Test
     public void testXMLEscapingWithoutUTF8() throws Exception {
         Renderer renderer = getRenderer();