diff --git a/src/main/java/org/json/XML.java b/src/main/java/org/json/XML.java index b3d05ebec..c76f971fe 100644 --- a/src/main/java/org/json/XML.java +++ b/src/main/java/org/json/XML.java @@ -156,33 +156,39 @@ public static String escape(String string) { /** * @param cp code point to test - * @return true if the code point is not valid for an XML + * @return true if the code point must be escaped when serializing to XML */ static boolean mustEscape(int cp) { + // Every ISO control character must be written as a numeric character + // reference when serializing. This includes #x9 (tab), #xA (LF) and + // #xD (CR): literal occurrences of these characters are subject to + // whitespace normalization by conforming XML parsers, so they can + // only survive a round trip as , and (see #1092). + // + // This method is only about the serialization path. Numeric character + // references found while parsing are validated with + // isValidXmlCharacter(int), which accepts the XML-allowed whitespace + // control characters (see #1059). + return Character.isISOControl(cp) || !isValidXmlCharacter(cp); + } + + /** + * @param cp code point to test + * @return true if the code point is a valid XML 1.0 character + */ + static boolean isValidXmlCharacter(int cp) { /* Valid range from https://www.w3.org/TR/REC-xml/#charsets * * #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] * * any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */ - // isISOControl is true when (cp >= 0 && cp <= 0x1F) || (cp >= 0x7F && cp <= 0x9F) - // all ISO control characters are out of range except tabs and new lines - return (Character.isISOControl(cp) - && cp != 0x9 - && cp != 0xA - && cp != 0xD - ) || !( - // Valid character range per W3C XML 1.0 spec: - // #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] - // Previously omitted #x9/#xA/#xD, causing unescape(" ") etc. - // to reject valid LF/TAB/CR as illegal (see #1059) - cp == 0x9 - || cp == 0xA - || cp == 0xD - || (cp >= 0x20 && cp <= 0xD7FF) - || (cp >= 0xE000 && cp <= 0xFFFD) - || (cp >= 0x10000 && cp <= 0x10FFFF) - ) + return cp == 0x9 + || cp == 0xA + || cp == 0xD + || (cp >= 0x20 && cp <= 0xD7FF) + || (cp >= 0xE000 && cp <= 0xFFFD) + || (cp >= 0x10000 && cp <= 0x10FFFF) ; } diff --git a/src/main/java/org/json/XMLTokener.java b/src/main/java/org/json/XMLTokener.java index dc90f84d4..41439e079 100644 --- a/src/main/java/org/json/XMLTokener.java +++ b/src/main/java/org/json/XMLTokener.java @@ -167,7 +167,11 @@ static String unescapeEntity(String e) throws JSONException { int cp = (e.charAt(1) == 'x' || e.charAt(1) == 'X') ? parseHexEntity(e) : parseDecimalEntity(e); - if (XML.mustEscape(cp)) { + // Validate against the XML 1.0 character range, not against + // XML.mustEscape: the serialization path escapes TAB/LF/CR, but + // numeric character references for them ( , , ) are + // valid XML and must be accepted here (see #1059 and #1092). + if (!XML.isValidXmlCharacter(cp)) { throw new JSONException("Invalid numeric character reference: &#" + e.substring(1) + ";"); } return new String(new int[] {cp}, 0, 1); diff --git a/src/test/java/org/json/junit/XMLTest.java b/src/test/java/org/json/junit/XMLTest.java index 0533673cf..a65abcb3d 100644 --- a/src/test/java/org/json/junit/XMLTest.java +++ b/src/test/java/org/json/junit/XMLTest.java @@ -1610,6 +1610,56 @@ public void testValidWhitespaceNumericEntityToJSONObject() { XML.toJSONObject(" "); } + /** + * Tests that serializing a string value containing literal TAB, LF and CR + * characters writes numeric character references (&#9;, &#10;, + * &#13;) instead of the literal characters. Literal occurrences of + * these characters are subject to whitespace normalization by conforming + * XML parsers, so the original value would be lost when the generated XML + * is read back. Regression test for #1092, where #1070 stopped + * XML.toString() and JSONML.toString() from escaping these characters. + */ + @Test + public void testEscapesLiteralWhitespaceControlCharsOnSerialization() { + // LF in a text node must survive as a numeric character reference + JSONObject jo = new JSONObject().put("value", "Lettuce\nTomato"); + String xmlOutput = XML.toString(jo); + assertTrue("XML should escape literal LF to preserve it, got: " + xmlOutput, + xmlOutput.contains(" ")); + + // TAB and CR likewise + JSONObject tabJo = new JSONObject().put("value", "a\tb\rc"); + String tabXml = XML.toString(tabJo); + assertTrue("XML should escape literal TAB, got: " + tabXml, tabXml.contains(" ")); + assertTrue("XML should escape literal CR, got: " + tabXml, tabXml.contains(" ")); + } + + /** + * Tests that the generated XML round-trips: the numeric character + * references produced for TAB/LF/CR are read back by + * {@code XML.toJSONObject} as the original characters. + */ + @Test + public void testWhitespaceControlCharsRoundTripThroughXml() { + String original = "Lettuce\nTomato\tMustard"; + JSONObject jo = new JSONObject().put("value", original); + String xmlOutput = XML.toString(jo); + JSONObject reparsed = XML.toJSONObject(xmlOutput); + assertEquals("LF/TAB must survive the XML round trip", + original, reparsed.getString("value")); + } + + /** + * Same escaping requirement for {@code JSONML.toString}. + */ + @Test + public void testJSONMLToStringEscapesLiteralNewlines() { + JSONObject jo = new JSONObject("{\"tagName\":\"MyTag\", \"value\":\"Lettuce\\nTomato\"}"); + String xmlOutput = JSONML.toString(jo); + assertTrue("JSONML should escape literal LF to preserve it, got: " + xmlOutput, + xmlOutput.contains(" ")); + } + }