diff --git a/docs/release.md b/docs/release.md index 1f439a9c..46198256 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,10 @@ ## Unreleased +* Fix compression and decompression of structured arrays containing datetime or + timedelta fields with Zstd, LZ4, and Blosc ({issue}`755`). Preserve record sizes + and reject object-containing records and read-only destination buffers. + (release_0.17.0)= * Allow building against a system Zlib. Note, this is only required if blosc is still diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 8c52dd95..7c1269d6 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -370,6 +370,8 @@ def decompress(source, dest=None): dest_1d = dest = PyBytes_FromStringAndSize(NULL, nbytes) else: dest_1d = ensure_contiguous_ndarray(dest) + if not dest_1d.flags.writeable: + raise ValueError("destination buffer is read-only") # obtain dest memoryview dest_mv = memoryview(dest_1d) diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index d1844e10..0909b08c 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -102,6 +102,17 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N if arr.dtype.kind in "Mm": arr = arr.view(np.int64) # type: ignore[arg-type] + # NumPy cannot export the buffer format of some structured dtypes, including + # datetime/timedelta fields. Keep each record's itemsize (used by Blosc's + # shuffle filter) and expose its bytes without copying or changing the input. + if isinstance(arr, np.ndarray) and arr.dtype.fields is not None: + if arr.dtype.hasobject: + raise TypeError("object arrays are not supported") + try: + memoryview(arr) + except ValueError: + arr = arr.view(np.dtype((np.void, arr.dtype.itemsize))) + # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: if flatten: diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 6eac246b..bb361cf3 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -166,6 +166,8 @@ def decompress(source, dest=None): dest_1d = dest = PyBytes_FromStringAndSize(NULL, dest_size) else: dest_1d = ensure_contiguous_ndarray(dest) + if not dest_1d.flags.writeable: + raise ValueError("destination buffer is read-only") # obtain dest memoryview dest_mv = memoryview(dest_1d) diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7b6ff3c0..59782f6d 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -245,6 +245,8 @@ def decompress(source, dest=None): dest_1d = dest = PyBytes_FromStringAndSize(NULL, dest_size) else: dest_1d = ensure_contiguous_ndarray(dest) + if not dest_1d.flags.writeable: + raise ValueError("destination buffer is read-only") # obtain dest memoryview dest_mv = memoryview(dest_1d) diff --git a/tests/test_structured_buffers.py b/tests/test_structured_buffers.py new file mode 100644 index 00000000..f9162972 --- /dev/null +++ b/tests/test_structured_buffers.py @@ -0,0 +1,145 @@ +import numpy as np +import pytest + +from numcodecs import LZ4, Blosc, Zstd, blosc +from numcodecs.compat import ensure_contiguous_ndarray + +temporal_dtypes = [ + np.dtype([('date', 'M8[D]'), ('value', 'i8')]), + np.dtype([('duration', '>m8[ns]'), ('value', 'M8[ms]'), ('duration', 'i4')], (2,))]), + np.dtype( + {'names': ['date', 'value'], 'formats': ['M8[D]', 'u1'], 'offsets': [1, 12], 'itemsize': 16} + ), +] +codecs = [Zstd(), LZ4(), Blosc(), Blosc(shuffle=Blosc.BITSHUFFLE)] + + +def make_array(dtype, shape, order): + arr = np.zeros(shape, dtype=dtype, order=order) + # Fill padding too, so raw-byte comparisons detect accidental field-wise copies. + raw = arr.reshape(-1, order='A').view('u1') + raw[:] = np.arange(raw.size, dtype='u1') + return arr + + +@pytest.mark.parametrize('dtype', temporal_dtypes) +@pytest.mark.parametrize('shape', [(), (0,), (2, 3)]) +@pytest.mark.parametrize('order', ['C', 'F']) +@pytest.mark.parametrize('flatten', [False, True]) +@pytest.mark.parametrize('writeable', [False, True]) +def test_structured_temporal_buffer_view(dtype, shape, order, flatten, writeable): + arr = make_array(dtype, shape, order) + arr.setflags(write=writeable) + expected = arr.tobytes(order='A') + result = ensure_contiguous_ndarray(arr, flatten=flatten) + + assert memoryview(result).tobytes(order='A') == expected + assert result.dtype.itemsize == dtype.itemsize + assert result.shape == ((arr.size,) if flatten else shape) + if not flatten: + assert result.strides == arr.strides + assert result.ctypes.data == arr.ctypes.data + assert result.flags.writeable == writeable + assert arr.dtype == dtype + assert arr.tobytes(order='A') == expected + + +@pytest.mark.parametrize('codec', codecs) +@pytest.mark.parametrize('dtype', temporal_dtypes) +@pytest.mark.parametrize('shape', [(), (2, 3)]) +@pytest.mark.parametrize('order', ['C', 'F']) +def test_structured_temporal_roundtrip(codec, dtype, shape, order): + arr = make_array(dtype, shape, order) + expected = arr.tobytes(order='A') + arr.setflags(write=False) + encoded = codec.encode(arr) + assert codec.decode(encoded) == expected + if isinstance(codec, Blosc): + typesize, _, _ = blosc._cbuffer_metainfo(encoded) + assert typesize == dtype.itemsize + + # An unaligned buffer is still contiguous and must be written without a copy. + backing = bytearray(arr.nbytes + 2) + out = np.ndarray(shape, dtype=dtype, buffer=backing, offset=1, order=order) + assert codec.decode(memoryview(encoded), out=out) is out + assert out.dtype == dtype + assert out.shape == shape + assert out.tobytes(order='A') == expected + assert backing[0] == backing[-1] == 0 + assert arr.dtype == dtype + assert arr.tobytes(order='A') == expected + + # Byte-format memoryviews remain valid inputs and outputs. + assert codec.decode(codec.encode(memoryview(expected))) == expected + raw_out = memoryview(bytearray(arr.nbytes)) + assert codec.decode(encoded, out=raw_out) is raw_out + assert raw_out.tobytes() == expected + + +@pytest.mark.parametrize('codec', codecs) +def test_issue_755(codec): + arr = np.array( + [('19000101', 123), ('19021201', 321)], dtype=[('date', 'datetime64[D]'), ('num', 'i8')] + ) + out = np.empty_like(arr) + assert codec.decode(codec.encode(arr), out=out) is out + np.testing.assert_array_equal(out, arr) + + +@pytest.mark.parametrize('codec', codecs) +@pytest.mark.parametrize('dtype', [np.dtype('i8'), temporal_dtypes[0]]) +@pytest.mark.parametrize('as_memoryview', [False, True]) +def test_decode_readonly_output(codec, dtype, as_memoryview): + arr = make_array(dtype, (3,), 'C') + encoded = codec.encode(arr.tobytes()) + out = np.zeros_like(arr) + expected = out.tobytes() + out.setflags(write=False) + if as_memoryview: + out = memoryview(out.view(f'V{out.dtype.itemsize}')) + with pytest.raises(ValueError, match='read-only'): + codec.decode(encoded, out=out) + assert out.tobytes() == expected + + +@pytest.mark.parametrize( + 'dtype', + [ + np.dtype([('object', 'O')]), + np.dtype([('nested', [('object', 'O'), ('date', 'M8[D]')])]), + np.dtype([('objects', 'O', (2,))]), + ], +) +@pytest.mark.parametrize('codec', codecs) +def test_structured_object_buffers_rejected(codec, dtype): + arr = np.zeros(3, dtype=dtype) + with pytest.raises(TypeError, match='object arrays are not supported'): + codec.encode(arr) + encoded = codec.encode(bytes(arr.nbytes)) + with pytest.raises(TypeError, match='object arrays are not supported'): + codec.decode(encoded, out=arr) + + +@pytest.mark.parametrize('codec', codecs) +def test_structured_temporal_buffer_guards(codec): + arr = make_array(temporal_dtypes[0], (6,), 'C') + with pytest.raises(ValueError, match='contiguous'): + codec.encode(arr[::2]) + encoded = codec.encode(arr.tobytes()) + with pytest.raises(ValueError, match='contiguous'): + codec.decode(encoded, out=np.zeros(12, dtype=arr.dtype)[::2]) + with pytest.raises(ValueError, match='destination buffer too small'): + codec.decode(encoded, out=np.zeros(5, dtype=arr.dtype)) + with pytest.raises(ValueError, match='Codec does not support buffers'): + ensure_contiguous_ndarray(arr, max_buffer_size=arr.nbytes - 1) + + +def test_regular_structured_buffer_unchanged(): + dtype = np.dtype([('nested', [('value', '>i8')]), ('tag', 'S3')]) + arr = make_array(dtype, (2, 3), 'F') + result = ensure_contiguous_ndarray(arr, flatten=False) + assert result.dtype == dtype + assert result is arr + assert ensure_contiguous_ndarray(memoryview(arr), flatten=False).dtype == dtype