Share repeated object keys within a parse (#1091)

src/main/java/org/json/JSONObject.java

@@ -276,7 +276,7 @@ private boolean parseJSONObject(JSONTokener jsonTokener, JSONParserConfiguration
                 return true;
             default:
                 obj = jsonTokener.nextSimpleValue(c);
-                key = obj.toString();
+                key = jsonTokener.shareKey(obj.toString(), jsonParserConfiguration.getMaxSharedKeys());
         }
 
         checkKeyForStrictMode(jsonTokener, jsonParserConfiguration, obj);

src/main/java/org/json/JSONParserConfiguration.java

@@ -4,6 +4,16 @@
  * Configuration object for the JSON parser. The configuration is immutable.
  */
 public class JSONParserConfiguration extends ParserConfiguration {
+    /**
+     * The default maximum number of distinct object keys shared within one parse.
+     */
+    public static final int DEFAULT_MAXIMUM_SHARED_KEYS = 1000;
+
+    /**
+     * Used to indicate that there is no limit on the number of distinct object keys shared within one parse.
+     */
+    public static final int UNLIMITED_SHARED_KEYS = -1;
+
     /**
      * Used to indicate whether to overwrite duplicate key or not.
      */
@@ -14,12 +24,18 @@ public class JSONParserConfiguration extends ParserConfiguration {
      */
     private boolean useNativeNulls;
 
+    /**
+     * The maximum number of distinct object keys shared within one parse.
+     */
+    private int maxSharedKeys;
+
     /**
      * Configuration with the default values.
      */
     public JSONParserConfiguration() {
         super();
         this.overwriteDuplicateKey = false;
+        this.maxSharedKeys = DEFAULT_MAXIMUM_SHARED_KEYS;
         // DO NOT DELETE THE FOLLOWING LINE -- it is used for strictMode testing
         // this.strictMode = true;
     }
@@ -38,6 +54,7 @@ protected JSONParserConfiguration clone() {
         clone.maxNestingDepth = maxNestingDepth;
         clone.keepStrings = keepStrings;
         clone.useNativeNulls = useNativeNulls;
+        clone.maxSharedKeys = maxSharedKeys;
         return clone;
     }
 
@@ -89,6 +106,24 @@ public JSONParserConfiguration withUseNativeNulls(final boolean useNativeNulls)
         return clone;
     }
 
+    /**
+     * Defines how many distinct object keys the parser shares within one parse. When a key repeats, for example
+     * in each element of an array of objects, the parser reuses the first String instance read for that key
+     * instead of keeping a separate copy in every object. Only the first <code>maxSharedKeys</code> distinct keys
+     * are shared, so input that begins with many unique keys may need a higher limit. The default is 1000.
+     * Parsed values are the same with any setting.
+     *
+     * @param maxSharedKeys the maximum number of distinct keys to share; 0 disables sharing, and
+     *        {@link #UNLIMITED_SHARED_KEYS} (or any negative value) removes the limit
+     * @return The existing configuration will not be modified. A new configuration is returned.
+     */
+    public JSONParserConfiguration withMaxSharedKeys(final int maxSharedKeys) {
+        JSONParserConfiguration clone = this.clone();
+        clone.maxSharedKeys = Math.max(maxSharedKeys, UNLIMITED_SHARED_KEYS);
+
+        return clone;
+    }
+
     /**
      * Sets the strict mode configuration for the JSON parser with default true value
      * <p>
@@ -140,6 +175,16 @@ public boolean isUseNativeNulls() {
     }
 
 
+    /**
+     * The maximum number of distinct object keys shared within one parse. 0 means keys are not shared,
+     * and {@link #UNLIMITED_SHARED_KEYS} means there is no limit.
+     *
+     * @return The <code>maxSharedKeys</code> configuration value.
+     */
+    public int getMaxSharedKeys() {
+        return this.maxSharedKeys;
+    }
+
     /**
      * The parser throws an Exception when strict mode is true and tries to parse invalid JSON characters.
      * Otherwise, the parser is more relaxed and might tolerate some invalid characters.

src/main/java/org/json/JSONTokener.java

@@ -2,6 +2,8 @@
 
 import java.io.*;
 import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
 
 /*
 Public Domain.
@@ -35,6 +37,9 @@ public class JSONTokener {
     // access to this object is required for strict mode checking
     private JSONParserConfiguration jsonParserConfiguration;
 
+    /** object keys already read, so repeated keys can share one String instance. */
+    private Map<String, String> sharedKeys;
+
     /**
      * Construct a JSONTokener from a Reader. The caller must close the Reader.
      *
@@ -124,6 +129,31 @@ void setJsonParserConfigurationInternal(JSONParserConfiguration jsonParserConfig
         this.jsonParserConfiguration = jsonParserConfiguration;
     }
 
+    /**
+     * Returns an earlier String instance equal to the key if this tokener has read one, otherwise the key itself.
+     *
+     * @param key an object key that was just read
+     * @param maxSharedKeys the maximum number of distinct keys to remember; 0 disables sharing and a negative
+     *        value means no limit
+     * @return a String equal to <code>key</code>
+     */
+    String shareKey(String key, int maxSharedKeys) {
+        if (maxSharedKeys == 0) {
+            return key;
+        }
+        if (this.sharedKeys == null) {
+            this.sharedKeys = new HashMap<String, String>();
+        }
+        String shared = this.sharedKeys.get(key);
+        if (shared != null) {
+            return shared;
+        }
+        if (maxSharedKeys < 0 || this.sharedKeys.size() < maxSharedKeys) {
+            this.sharedKeys.put(key, key);
+        }
+        return key;
+    }
+
     /**
      * Back up one character. This provides a sort of lookahead capability,
      * so that you can test for a digit or letter before attempting to parse

src/test/java/org/json/junit/JSONParserConfigurationTest.java

@@ -16,9 +16,11 @@
 import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 public class JSONParserConfigurationTest {
@@ -708,4 +710,140 @@ private List<String> getNonCompliantJSONObjectList() {
         );
     }
 
+    private static final String SHARED_KEYS_SOURCE = "{\"rows\": ["
+            + "{\"name\": \"a\", \"count\": 1, \"big\": 12345678901, \"ratio\": 1.5, \"flag\": true, \"missing\": null,"
+            + " \"nested\": {\"name\": \"inner\", \"list\": [1, \"name\", null]}},"
+            + "{\"name\": \"name\", \"count\": 2, \"nested\": {\"name\": \"inner2\", \"list\": []}},"
+            + "{\"name\": \"third\", \"count\": -3}"
+            + "]}";
+
+    @Test
+    public void testMaxSharedKeysConfig() {
+        JSONParserConfiguration config = new JSONParserConfiguration();
+        assertEquals(JSONParserConfiguration.DEFAULT_MAXIMUM_SHARED_KEYS, config.getMaxSharedKeys());
+        assertEquals(1000, config.getMaxSharedKeys());
+        assertEquals(0, config.withMaxSharedKeys(0).getMaxSharedKeys());
+        assertEquals(JSONParserConfiguration.UNLIMITED_SHARED_KEYS, config.withMaxSharedKeys(-1).getMaxSharedKeys());
+        assertEquals(JSONParserConfiguration.UNLIMITED_SHARED_KEYS, config.withMaxSharedKeys(-5).getMaxSharedKeys());
+
+        JSONParserConfiguration chained = config.withMaxSharedKeys(7).withOverwriteDuplicateKey(true).withStrictMode(false);
+        assertEquals(7, chained.getMaxSharedKeys());
+        assertTrue(chained.isOverwriteDuplicateKey());
+        assertEquals(1000, config.getMaxSharedKeys());
+    }
+
+    @Test
+    public void testSharedKeysByDefault() {
+        JSONArray rows = new JSONObject(SHARED_KEYS_SOURCE).getJSONArray("rows");
+        String nameKey = key(rows.getJSONObject(0), "name");
+        for (int i = 1; i < rows.length(); i++) {
+            assertSame(nameKey, key(rows.getJSONObject(i), "name"));
+        }
+        assertSame(nameKey, key(rows.getJSONObject(0).getJSONObject("nested"), "name"));
+        assertSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(2), "count"));
+        assertSame(key(rows.getJSONObject(0).getJSONObject("nested"), "list"),
+                key(rows.getJSONObject(1).getJSONObject("nested"), "list"));
+
+        // string values equal to a key are not shared
+        assertNotSame(nameKey, rows.getJSONObject(1).get("name"));
+        assertNotSame(nameKey, rows.getJSONObject(0).getJSONObject("nested").getJSONArray("list").get(1));
+    }
+
+    @Test
+    public void testSharedKeysDisabled() {
+        JSONArray rows = new JSONObject(SHARED_KEYS_SOURCE, new JSONParserConfiguration().withMaxSharedKeys(0))
+                .getJSONArray("rows");
+        assertNotSame(key(rows.getJSONObject(0), "name"), key(rows.getJSONObject(1), "name"));
+        assertNotSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(1), "count"));
+    }
+
+    @Test
+    public void testSharedKeysLimit() {
+        JSONArray rows = new JSONArray("[{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 4, \"b\": 5, \"c\": 6}]",
+                new JSONParserConfiguration().withMaxSharedKeys(2));
+        assertSame(key(rows.getJSONObject(0), "a"), key(rows.getJSONObject(1), "a"));
+        assertSame(key(rows.getJSONObject(0), "b"), key(rows.getJSONObject(1), "b"));
+        assertNotSame(key(rows.getJSONObject(0), "c"), key(rows.getJSONObject(1), "c"));
+    }
+
+    @Test
+    public void testSharedKeysUnlimited() {
+        int keyCount = JSONParserConfiguration.DEFAULT_MAXIMUM_SHARED_KEYS + 500;
+        StringBuilder row = new StringBuilder("{");
+        for (int i = 0; i < keyCount; i++) {
+            row.append(i == 0 ? "" : ",").append("\"k").append(i).append("\":").append(i);
+        }
+        row.append('}');
+        String source = "[" + row + "," + row + "]";
+
+        JSONArray unlimited = new JSONArray(source,
+                new JSONParserConfiguration().withMaxSharedKeys(JSONParserConfiguration.UNLIMITED_SHARED_KEYS));
+        for (int i = 0; i < keyCount; i++) {
+            assertSame(key(unlimited.getJSONObject(0), "k" + i), key(unlimited.getJSONObject(1), "k" + i));
+        }
+
+        JSONArray limited = new JSONArray(source);
+        String lastKey = "k" + (keyCount - 1);
+        assertSame(key(limited.getJSONObject(0), "k0"), key(limited.getJSONObject(1), "k0"));
+        assertNotSame(key(limited.getJSONObject(0), lastKey), key(limited.getJSONObject(1), lastKey));
+        assertTrue(unlimited.similar(limited));
+    }
+
+    @Test
+    public void testSharedKeysNotSharedAcrossParses() {
+        JSONObject first = new JSONObject("{\"name\": 1}");
+        JSONObject second = new JSONObject("{\"name\": 2}");
+        assertNotSame(key(first, "name"), key(second, "name"));
+    }
+
+    @Test
+    public void testSharedKeysParseUnchanged() {
+        JSONObject unshared = new JSONObject(SHARED_KEYS_SOURCE, new JSONParserConfiguration().withMaxSharedKeys(0));
+        JSONObject shared = new JSONObject(SHARED_KEYS_SOURCE);
+        JSONObject unlimited = new JSONObject(SHARED_KEYS_SOURCE,
+                new JSONParserConfiguration().withMaxSharedKeys(JSONParserConfiguration.UNLIMITED_SHARED_KEYS));
+        assertTrue(unshared.similar(shared));
+        assertTrue(unshared.similar(unlimited));
+        assertEquals(unshared.toString(), shared.toString());
+        assertEquals(unshared.toString(), unlimited.toString());
+
+        JSONObject row = shared.getJSONArray("rows").getJSONObject(0);
+        assertEquals(Integer.class, row.get("count").getClass());
+        assertEquals(Long.class, row.get("big").getClass());
+        assertEquals(java.math.BigDecimal.class, row.get("ratio").getClass());
+        assertEquals(Boolean.TRUE, row.get("flag"));
+        assertSame(JSONObject.NULL, row.get("missing"));
+        assertSame(JSONObject.NULL, row.getJSONObject("nested").getJSONArray("list").get(2));
+        assertEquals("third", shared.getJSONArray("rows").getJSONObject(2).get("name"));
+    }
+
+    @Test
+    public void testSharedKeysLenientSyntax() {
+        JSONArray rows = new JSONArray("[{\"name\": 1, \"count\": 2}, {'name': 'single', count: 3}]",
+                new JSONParserConfiguration().withStrictMode(false));
+        assertSame(key(rows.getJSONObject(0), "name"), key(rows.getJSONObject(1), "name"));
+        assertSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(1), "count"));
+        assertEquals("single", rows.getJSONObject(1).get("name"));
+    }
+
+    @Test
+    public void testSharedKeysStrictMode() {
+        JSONArray rows = new JSONArray("[{\"a\": 1}, {\"a\": 2}]", new JSONParserConfiguration().withStrictMode());
+        assertSame(key(rows.getJSONObject(0), "a"), key(rows.getJSONObject(1), "a"));
+        assertFalse(rows.getJSONObject(0).similar(rows.getJSONObject(1)));
+    }
+
+    /**
+     * Returns the key instance a JSONObject holds, rather than the caller's equal String.
+     */
+    private static String key(JSONObject jsonObject, String name) {
+        for (String key : jsonObject.keySet()) {
+            if (key.equals(name)) {
+                return key;
+            }
+        }
+        fail("missing key " + name);
+        return null;
+    }
+
 }