Escape TAB/LF/CR again when serializing to XML

src/main/java/org/json/XML.java

@@ -156,33 +156,39 @@ public static String escape(String string) {
 
     /**
      * @param cp code point to test
-     * @return true if the code point is not valid for an XML
+     * @return true if the code point must be escaped when serializing to XML
      */
     static boolean mustEscape(int cp) {
+        // Every ISO control character must be written as a numeric character
+        // reference when serializing. This includes #x9 (tab), #xA (LF) and
+        // #xD (CR): literal occurrences of these characters are subject to
+        // whitespace normalization by conforming XML parsers, so they can
+        // only survive a round trip as 	, 
 and 
 (see #1092).
+        //
+        // This method is only about the serialization path. Numeric character
+        // references found while parsing are validated with
+        // isValidXmlCharacter(int), which accepts the XML-allowed whitespace
+        // control characters (see #1059).
+        return Character.isISOControl(cp) || !isValidXmlCharacter(cp);
+    }
+
+    /**
+     * @param cp code point to test
+     * @return true if the code point is a valid XML 1.0 character
+     */
+    static boolean isValidXmlCharacter(int cp) {
         /* Valid range from https://www.w3.org/TR/REC-xml/#charsets
          *
          * #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
          *
          * any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
          */
-        // isISOControl is true when (cp >= 0 && cp <= 0x1F) || (cp >= 0x7F && cp <= 0x9F)
-        // all ISO control characters are out of range except tabs and new lines
-        return (Character.isISOControl(cp)
-                && cp != 0x9
-                && cp != 0xA
-                && cp != 0xD
-            ) || !(
-                // Valid character range per W3C XML 1.0 spec:
-                // #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
-                // Previously omitted #x9/#xA/#xD, causing unescape("&#10;") etc.
-                // to reject valid LF/TAB/CR as illegal (see #1059)
-                cp == 0x9
-                || cp == 0xA
-                || cp == 0xD
-                || (cp >= 0x20 && cp <= 0xD7FF)
-                || (cp >= 0xE000 && cp <= 0xFFFD)
-                || (cp >= 0x10000 && cp <= 0x10FFFF)
-            )
+        return cp == 0x9
+            || cp == 0xA
+            || cp == 0xD
+            || (cp >= 0x20 && cp <= 0xD7FF)
+            || (cp >= 0xE000 && cp <= 0xFFFD)
+            || (cp >= 0x10000 && cp <= 0x10FFFF)
         ;
     }
 

src/main/java/org/json/XMLTokener.java

@@ -167,7 +167,11 @@ static String unescapeEntity(String e) throws JSONException {
             int cp = (e.charAt(1) == 'x' || e.charAt(1) == 'X')
                 ? parseHexEntity(e)
                 : parseDecimalEntity(e);
-            if (XML.mustEscape(cp)) {
+            // Validate against the XML 1.0 character range, not against
+            // XML.mustEscape: the serialization path escapes TAB/LF/CR, but
+            // numeric character references for them (	, 
, 
) are
+            // valid XML and must be accepted here (see #1059 and #1092).
+            if (!XML.isValidXmlCharacter(cp)) {
                 throw new JSONException("Invalid numeric character reference: &#" + e.substring(1) + ";");
             }
             return new String(new int[] {cp}, 0, 1);

src/test/java/org/json/junit/XMLTest.java

@@ -1610,6 +1610,56 @@ public void testValidWhitespaceNumericEntityToJSONObject() {
         XML.toJSONObject("<a>&#13;</a>");
     }
 
+    /**
+     * Tests that serializing a string value containing literal TAB, LF and CR
+     * characters writes numeric character references (&amp;#9;, &amp;#10;,
+     * &amp;#13;) instead of the literal characters. Literal occurrences of
+     * these characters are subject to whitespace normalization by conforming
+     * XML parsers, so the original value would be lost when the generated XML
+     * is read back. Regression test for #1092, where #1070 stopped
+     * XML.toString() and JSONML.toString() from escaping these characters.
+     */
+    @Test
+    public void testEscapesLiteralWhitespaceControlCharsOnSerialization() {
+        // LF in a text node must survive as a numeric character reference
+        JSONObject jo = new JSONObject().put("value", "Lettuce\nTomato");
+        String xmlOutput = XML.toString(jo);
+        assertTrue("XML should escape literal LF to preserve it, got: " + xmlOutput,
+                xmlOutput.contains("&#xa;"));
+
+        // TAB and CR likewise
+        JSONObject tabJo = new JSONObject().put("value", "a\tb\rc");
+        String tabXml = XML.toString(tabJo);
+        assertTrue("XML should escape literal TAB, got: " + tabXml, tabXml.contains("&#x9;"));
+        assertTrue("XML should escape literal CR, got: " + tabXml, tabXml.contains("&#xd;"));
+    }
+
+    /**
+     * Tests that the generated XML round-trips: the numeric character
+     * references produced for TAB/LF/CR are read back by
+     * {@code XML.toJSONObject} as the original characters.
+     */
+    @Test
+    public void testWhitespaceControlCharsRoundTripThroughXml() {
+        String original = "Lettuce\nTomato\tMustard";
+        JSONObject jo = new JSONObject().put("value", original);
+        String xmlOutput = XML.toString(jo);
+        JSONObject reparsed = XML.toJSONObject(xmlOutput);
+        assertEquals("LF/TAB must survive the XML round trip",
+                original, reparsed.getString("value"));
+    }
+
+    /**
+     * Same escaping requirement for {@code JSONML.toString}.
+     */
+    @Test
+    public void testJSONMLToStringEscapesLiteralNewlines() {
+        JSONObject jo = new JSONObject("{\"tagName\":\"MyTag\", \"value\":\"Lettuce\\nTomato\"}");
+        String xmlOutput = JSONML.toString(jo);
+        assertTrue("JSONML should escape literal LF to preserve it, got: " + xmlOutput,
+                xmlOutput.contains("&#xa;"));
+    }
+
 }