From 3b93afd4e10795a9bd8bf08fdba2b976cb2bfe87 Mon Sep 17 00:00:00 2001 From: Lyin-OTTF Date: Tue, 20 Oct 2026 13:29:07 +0000 Subject: [PATCH] Escape TAB/LF/CR again when serializing to XML XML.mustEscape() is shared by XML.escape() (serialization) and XMLTokener.unescapeEntity() (parsing), but the two callers need opposite answers for #x9/#xA/#xD. #1070 added those codepoints to the valid-range clause to fix unescape(" "), which also stopped XML.toString() and JSONML.toString() from escaping literal TAB, LF and CR. Literal occurrences of these characters are subject to whitespace normalization by conforming XML parsers, so a value like "Lettuce\nTomato" loses its line breaks when the generated XML is read back by any standard parser. This regressed #1092. Split the two concerns: mustEscape() returns true for every ISO control character again (serialization), and the new isValidXmlCharacter() checks the XML 1.0 character range for numeric character references during parsing. unescapeEntity() now validates with isValidXmlCharacter(), so , and keep being accepted (#1059/#1070 behavior unchanged). Fixes #1092 --- src/main/java/org/json/XML.java | 44 +++++++++++--------- src/main/java/org/json/XMLTokener.java | 6 ++- src/test/java/org/json/junit/XMLTest.java | 50 +++++++++++++++++++++++ 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/json/XML.java b/src/main/java/org/json/XML.java index b3d05ebec..c76f971fe 100644 --- a/src/main/java/org/json/XML.java +++ b/src/main/java/org/json/XML.java @@ -156,33 +156,39 @@ public static String escape(String string) { /** * @param cp code point to test - * @return true if the code point is not valid for an XML + * @return true if the code point must be escaped when serializing to XML */ static boolean mustEscape(int cp) { + // Every ISO control character must be written as a numeric character + // reference when serializing. This includes #x9 (tab), #xA (LF) and + // #xD (CR): literal occurrences of these characters are subject to + // whitespace normalization by conforming XML parsers, so they can + // only survive a round trip as , and (see #1092). + // + // This method is only about the serialization path. Numeric character + // references found while parsing are validated with + // isValidXmlCharacter(int), which accepts the XML-allowed whitespace + // control characters (see #1059). + return Character.isISOControl(cp) || !isValidXmlCharacter(cp); + } + + /** + * @param cp code point to test + * @return true if the code point is a valid XML 1.0 character + */ + static boolean isValidXmlCharacter(int cp) { /* Valid range from https://www.w3.org/TR/REC-xml/#charsets * * #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] * * any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */ - // isISOControl is true when (cp >= 0 && cp <= 0x1F) || (cp >= 0x7F && cp <= 0x9F) - // all ISO control characters are out of range except tabs and new lines - return (Character.isISOControl(cp) - && cp != 0x9 - && cp != 0xA - && cp != 0xD - ) || !( - // Valid character range per W3C XML 1.0 spec: - // #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] - // Previously omitted #x9/#xA/#xD, causing unescape(" ") etc. - // to reject valid LF/TAB/CR as illegal (see #1059) - cp == 0x9 - || cp == 0xA - || cp == 0xD - || (cp >= 0x20 && cp <= 0xD7FF) - || (cp >= 0xE000 && cp <= 0xFFFD) - || (cp >= 0x10000 && cp <= 0x10FFFF) - ) + return cp == 0x9 + || cp == 0xA + || cp == 0xD + || (cp >= 0x20 && cp <= 0xD7FF) + || (cp >= 0xE000 && cp <= 0xFFFD) + || (cp >= 0x10000 && cp <= 0x10FFFF) ; } diff --git a/src/main/java/org/json/XMLTokener.java b/src/main/java/org/json/XMLTokener.java index dc90f84d4..41439e079 100644 --- a/src/main/java/org/json/XMLTokener.java +++ b/src/main/java/org/json/XMLTokener.java @@ -167,7 +167,11 @@ static String unescapeEntity(String e) throws JSONException { int cp = (e.charAt(1) == 'x' || e.charAt(1) == 'X') ? parseHexEntity(e) : parseDecimalEntity(e); - if (XML.mustEscape(cp)) { + // Validate against the XML 1.0 character range, not against + // XML.mustEscape: the serialization path escapes TAB/LF/CR, but + // numeric character references for them ( , , ) are + // valid XML and must be accepted here (see #1059 and #1092). + if (!XML.isValidXmlCharacter(cp)) { throw new JSONException("Invalid numeric character reference: &#" + e.substring(1) + ";"); } return new String(new int[] {cp}, 0, 1); diff --git a/src/test/java/org/json/junit/XMLTest.java b/src/test/java/org/json/junit/XMLTest.java index 0533673cf..a65abcb3d 100644 --- a/src/test/java/org/json/junit/XMLTest.java +++ b/src/test/java/org/json/junit/XMLTest.java @@ -1610,6 +1610,56 @@ public void testValidWhitespaceNumericEntityToJSONObject() { XML.toJSONObject(" "); } + /** + * Tests that serializing a string value containing literal TAB, LF and CR + * characters writes numeric character references (&#9;, &#10;, + * &#13;) instead of the literal characters. Literal occurrences of + * these characters are subject to whitespace normalization by conforming + * XML parsers, so the original value would be lost when the generated XML + * is read back. Regression test for #1092, where #1070 stopped + * XML.toString() and JSONML.toString() from escaping these characters. + */ + @Test + public void testEscapesLiteralWhitespaceControlCharsOnSerialization() { + // LF in a text node must survive as a numeric character reference + JSONObject jo = new JSONObject().put("value", "Lettuce\nTomato"); + String xmlOutput = XML.toString(jo); + assertTrue("XML should escape literal LF to preserve it, got: " + xmlOutput, + xmlOutput.contains(" ")); + + // TAB and CR likewise + JSONObject tabJo = new JSONObject().put("value", "a\tb\rc"); + String tabXml = XML.toString(tabJo); + assertTrue("XML should escape literal TAB, got: " + tabXml, tabXml.contains(" ")); + assertTrue("XML should escape literal CR, got: " + tabXml, tabXml.contains(" ")); + } + + /** + * Tests that the generated XML round-trips: the numeric character + * references produced for TAB/LF/CR are read back by + * {@code XML.toJSONObject} as the original characters. + */ + @Test + public void testWhitespaceControlCharsRoundTripThroughXml() { + String original = "Lettuce\nTomato\tMustard"; + JSONObject jo = new JSONObject().put("value", original); + String xmlOutput = XML.toString(jo); + JSONObject reparsed = XML.toJSONObject(xmlOutput); + assertEquals("LF/TAB must survive the XML round trip", + original, reparsed.getString("value")); + } + + /** + * Same escaping requirement for {@code JSONML.toString}. + */ + @Test + public void testJSONMLToStringEscapesLiteralNewlines() { + JSONObject jo = new JSONObject("{\"tagName\":\"MyTag\", \"value\":\"Lettuce\\nTomato\"}"); + String xmlOutput = JSONML.toString(jo); + assertTrue("JSONML should escape literal LF to preserve it, got: " + xmlOutput, + xmlOutput.contains(" ")); + } + }