Keep the fields of structured dtypes in the JSON codec

docs/release.md

@@ -19,6 +19,10 @@
 * Allow building against a system Zlib. Note, this is only required if blosc is still
   vendored, and a vendored Zlib is no longer built if blosc is not.
   By {user}`Elliott Sales de Andrade <QuLogic>`, {issue}`860`
+* The `JSON` codec now keeps the fields of structured dtypes, so structured arrays
+  (including ones with object fields) can be decoded. Before, the dtype was stored as
+  `|V<n>` and decoding failed.
+  By {user}`Raashish Aggarwal <raashish1601>`, {issue}`518`
 
 ## 0.17.0
 

src/numcodecs/json.py

@@ -71,17 +71,30 @@ def encode(self, buf):
         except ValueError:  # pragma: no cover
             buf = np.asarray(buf, dtype=object)
         items = np.atleast_1d(buf).tolist()
-        items.append(buf.dtype.str)
+        if buf.dtype.fields is not None:
+            # tolist() leaves subarray fields as arrays
+            items = _arrays_to_lists(items)
+            # dtype.str is only "|V<n>" for structured dtypes, so store the fields
+            items.append(buf.dtype.descr)
+        else:
+            items.append(buf.dtype.str)
         items.append(buf.shape)
         return self._encoder.encode(items).encode(self._text_encoding)
 
     def decode(self, buf, out=None):
         items = self._decoder.decode(ensure_text(buf, self._text_encoding))
-        dec = np.empty(items[-1], dtype=items[-2])
+        dtype = items[-2]
+        if isinstance(dtype, list):
+            # structured dtype, encoded as dtype.descr (JSON turned tuples into lists)
+            dtype = np.dtype(_descr_from_json(dtype))
+            values = _records_from_json(items[:-2], dtype, max(len(items[-1]), 1))
+        else:
+            values = items[:-2]
+        dec = np.empty(items[-1], dtype=dtype)
         if not items[-1]:
-            dec[...] = items[0]
+            dec[...] = values[0]
         else:
-            dec[:] = items[:-2]
+            dec[:] = values
         if out is not None:
             np.copyto(out, dec)
             return out
@@ -105,3 +118,40 @@ def __repr__(self):
         return textwrap.fill(
             f'{classname}({params})', width=80, break_long_words=False, subsequent_indent='     '
         )
+
+
+def _arrays_to_lists(values):
+    """Recursively replace the arrays left by ``tolist()`` on a structured array."""
+    if isinstance(values, np.ndarray):
+        values = values.tolist()
+    if isinstance(values, (list, tuple)):
+        return [_arrays_to_lists(v) for v in values]
+    return values
+
+
+def _descr_from_json(descr):
+    """Rebuild the list of tuples of a structured ``dtype.descr`` decoded from JSON."""
+    fields = []
+    for name, field_type, *shape in descr:
+        if isinstance(field_type, list):
+            field_type = _descr_from_json(field_type)
+        fields.append((name, field_type, *(tuple(s) for s in shape)))
+    return fields
+
+
+def _records_from_json(values, dtype, depth=0):
+    """Turn the records of a structured array decoded from JSON back into tuples.
+
+    ``depth`` is the number of array dimensions above the records.
+    """
+    if depth > 0:
+        return [_records_from_json(v, dtype, depth - 1) for v in values]
+    if dtype.fields is not None:
+        return tuple(
+            _records_from_json(v, dtype.fields[name][0])
+            for name, v in zip(dtype.names, values, strict=True)
+        )
+    if dtype.subdtype is not None and dtype.subdtype[0].fields is not None:
+        base, shape = dtype.subdtype
+        return _records_from_json(values, base, len(shape))
+    return values

tests/test_json.py

@@ -83,3 +83,28 @@ def test_non_numpy_inputs(input_data, dtype):
     for codec in codecs:
         output_data = codec.decode(codec.encode(data))
         assert input_data == output_data.tolist()
+
+
+@pytest.mark.parametrize(
+    "arr",
+    [
+        # from #518
+        np.array(
+            [
+                (1, 'dataset_1', {'source': '.', 'path': '/dataset_1', 'object_id': None}),
+                (2, 'dataset_2', {'source': '.', 'path': '/dataset_2', 'object_id': None}),
+            ],
+            dtype=[('id', '<i4'), ('name', 'O'), ('reference', 'O')],
+        ),
+        np.array((1, 2.5), dtype=[('x', '<i8'), ('y', '<f4')]),
+        np.arange(12).astype('f8').view([('a', '<f8'), ('b', '<f8', (2,))]).reshape(2, 2),
+        np.zeros(3, dtype=[('outer', [('p', '<i4'), ('q', '<f8')]), ('z', '<u1')]),
+        np.zeros(2, dtype=[('inner', [('p', '<i4')], (2,))]),
+    ],
+)
+def test_structured_dtype(arr):
+    codec = JSON()
+    dec = codec.decode(codec.encode(arr))
+    assert dec.dtype == arr.dtype
+    assert dec.shape == arr.shape
+    np.testing.assert_equal(dec.tolist(), arr.tolist())