From e3a27e9d80eb23acc733d3bfb74601f779261076 Mon Sep 17 00:00:00 2001 From: Gaelle Letort Date: Mon, 24 Aug 2026 10:59:57 +0200 Subject: [PATCH 1/7] Handle ndarray types in short notation with bytes --- src/appose/shm.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index 528c4af..8e395a8 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -188,8 +188,23 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None: def _bytes_per_element(dtype: str) -> int | float: + """ + Returns the number of bytes for the given type name + The type name can be standard, e.g. uint16, float32... in that case, parsing the number in the string gives the number of bits, to divide by 8. + The type name can also be a short version with bytes numbers, e.g. >u2, u2', ' Date: Wed, 23 Sep 2026 12:10:59 -0500 Subject: [PATCH 2/7] Improve _bytes_per_element docstring --- src/appose/shm.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index 8e395a8..fb2c0fa 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -189,9 +189,13 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None: def _bytes_per_element(dtype: str) -> int | float: """ - Returns the number of bytes for the given type name - The type name can be standard, e.g. uint16, float32... in that case, parsing the number in the string gives the number of bits, to divide by 8. - The type name can also be a short version with bytes numbers, e.g. >u2, u2 or Date: Wed, 23 Sep 2026 12:11:11 -0500 Subject: [PATCH 3/7] Improve _bytes_per_element comments --- src/appose/shm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index fb2c0fa..0cee3b0 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -198,15 +198,15 @@ def _bytes_per_element(dtype: str) -> int | float: the number of bytes, so the parsed value is returned as is. """ try: - ## boolean object should be one byte if dtype.startswith("bool"): + # 1-byte boolean object bytes_size = 1 - # Standard names (e.g., 'uint16', 'float32') elif dtype.startswith(("uint", "int", "float", "complex")): + # standard names (e.g. 'uint16', 'float32') bits = int(re.sub("[^0-9]", "", dtype)) bytes_size = bits / 8 - # Short names (e.g., '>u2', 'u2', ' Date: Wed, 23 Sep 2026 13:55:31 -0500 Subject: [PATCH 4/7] Normalize NDArray dtypes to standard names NDArray now accepts NumPy-style short forms (e.g. u2, f4, |u1, =c8) in addition to standard names, normalizing them to the standard name (e.g. uint16), so that only standard names cross the IPC channel. Only platform-independent types are supported, via a fixed lookup table, so that parsing behaves the same in every environment. Explicit byte orders (< or >) are rejected: array data is always native order. See also apposed/appose-java@655db12e865803c21c4c579c952c2a183632a2a8. Co-Authored-By: Claude Opus 5.5 --- src/appose/shm.py | 96 +++++++++++++++++++++++++++++++++++------------ tests/test_shm.py | 86 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 24 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index 0cee3b0..bb6dc79 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -8,8 +8,7 @@ from __future__ import annotations -import re -from math import ceil, prod +from math import prod from multiprocessing import resource_tracker, shared_memory from typing import TYPE_CHECKING @@ -129,15 +128,19 @@ def __init__(self, dtype: str, shape: list[int], shm: SharedMemory | None = None Args: dtype: The type of the data elements; e.g. int8, uint8, float32, float64. + NumPy-style short forms (e.g. u2, f4, |u1, =c8) are also accepted, + and normalized to the standard name (e.g. uint16, float32). + Explicit byte orders (< or >) are rejected: Appose arrays + always use the machine's native byte order. shape: The dimensional extents; e.g. a stack of 7 image planes with resolution 512x512 would have shape [7, 512, 512]. shm: The SharedMemory containing the array data, or None to create it. """ - self.dtype: str = dtype + self.dtype: str = _normalize_dtype(dtype) self.shape: list[int] = shape self.shm: SharedMemory = ( SharedMemory( - create=True, rsize=ceil(prod(shape) * _bytes_per_element(dtype)) + create=True, rsize=prod(shape) * _bytes_per_element(self.dtype) ) if shm is None else shm @@ -187,28 +190,73 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None: ) -def _bytes_per_element(dtype: str) -> int | float: +# Standard dtype names, with the number of bytes per element of each. +_DTYPE_SIZES = { + "int8": 1, + "int16": 2, + "int32": 4, + "int64": 8, + "uint8": 1, + "uint16": 2, + "uint32": 4, + "uint64": 8, + "float16": 2, + "float32": 4, + "float64": 8, + "complex64": 8, + "complex128": 16, + "bool": 1, +} + +# NumPy-style short forms of the standard dtype names. +_DTYPE_ALIASES = { + "i1": "int8", + "i2": "int16", + "i4": "int32", + "i8": "int64", + "u1": "uint8", + "u2": "uint16", + "u4": "uint32", + "u8": "uint64", + "f2": "float16", + "f4": "float32", + "f8": "float64", + "c8": "complex64", + "c16": "complex128", + "b1": "bool", + "?": "bool", +} + + +def _normalize_dtype(dtype: str) -> str: """ - Return the number of bytes for the given type name. + Return the standard name of the given dtype; e.g. " "uint16". - * For long-form types (e.g. uint16 or float32), the string indicates - the number of bits, so the parsed value is divided by 8. + Accepts standard names (e.g. uint16, float32) as well as NumPy-style + short forms (e.g. u2, f4). A short form may be prefixed with = (native + byte order) or | (byte order not applicable), which is ignored. Explicit + byte orders (< or >) are rejected, so that parsing behaves the same on + every machine; Appose arrays always use the machine's native byte order. - * For short-form types (e.g. >u2 or u2', '")): + raise ValueError( + f"Unsupported dtype: {dtype} " + "(Appose arrays are always native byte order; " + "omit the < or > prefix)" + ) + short = dtype[1:] if dtype.startswith(("=", "|")) else dtype + if short not in _DTYPE_ALIASES: + raise ValueError(f"Unsupported dtype: {dtype}") + return _DTYPE_ALIASES[short] + - return bytes_size +def _bytes_per_element(dtype: str) -> int: + """ + Return the number of bytes per element for the given dtype. + """ + return _DTYPE_SIZES[_normalize_dtype(dtype)] diff --git a/tests/test_shm.py b/tests/test_shm.py index 7c50093..9f628d8 100644 --- a/tests/test_shm.py +++ b/tests/test_shm.py @@ -2,8 +2,11 @@ # Copyright (C) 2023 - 2026 Appose developers. # SPDX-License-Identifier: BSD-2-Clause +import pytest + import appose from appose.service import TaskStatus +from appose.shm import _bytes_per_element, _normalize_dtype ndarray_inspect = """ task.outputs["rsize"] = data.shm.rsize @@ -39,3 +42,86 @@ def test_ndarray(): assert "uint16" == task.outputs["dtype"] assert [2, 20, 25] == task.outputs["shape"] assert 123 + 78 + 210 == task.outputs["sum"] + + +def test_dtype_standard_names(): + for dtype, size in [ + ("int8", 1), + ("int16", 2), + ("int32", 4), + ("int64", 8), + ("uint8", 1), + ("uint16", 2), + ("uint32", 4), + ("uint64", 8), + ("float16", 2), + ("float32", 4), + ("float64", 8), + ("complex64", 8), + ("complex128", 16), + ("bool", 1), + ]: + assert dtype == _normalize_dtype(dtype) + assert size == _bytes_per_element(dtype) + + +def test_dtype_short_forms(): + for short, name in [ + ("i1", "int8"), + ("i2", "int16"), + ("i4", "int32"), + ("i8", "int64"), + ("u1", "uint8"), + ("u2", "uint16"), + ("u4", "uint32"), + ("u8", "uint64"), + ("f2", "float16"), + ("f4", "float32"), + ("f8", "float64"), + ("c8", "complex64"), + ("c16", "complex128"), + ("b1", "bool"), + ("?", "bool"), + ]: + assert name == _normalize_dtype(short) + assert name == _normalize_dtype("=" + short) + assert name == _normalize_dtype("|" + short) + + +def test_dtype_explicit_byte_order(): + for dtype in ["u2", "c16", "float32"]: + with pytest.raises(ValueError, match="native byte order"): + _normalize_dtype(dtype) + + +def test_dtype_unsupported(): + for dtype in [ + "", + "=", + "==u2", + "|=u2", + "=uint16", + "|uint8", + "uint", + "u3", + "f16", + "c32", + "?1", + "l", + "g", + "longdouble", + "float128", + "intp", + "U10", + "datetime64[ns]", + "object", + "FLOAT32", + ]: + with pytest.raises(ValueError, match="Unsupported dtype"): + _normalize_dtype(dtype) + + +def test_ndarray_normalizes_dtype(): + with appose.NDArray("=u2", [3, 5]) as data: + assert "uint16" == data.dtype + assert 3 * 5 * 2 == data.shm.rsize From e85180bf48e3d12dcac1cc8b8678b53c5e80024b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 Sep 2026 15:47:49 -0500 Subject: [PATCH 5/7] Add NDArray.from_ndarray to copy a NumPy array It allocates shared memory matching the array's dtype and shape, then copies the data by assignment, so that NumPy converts values into the native byte order and C-ordered layout. This handles e.g. big-endian images from file readers, whose str(dtype) is '>u2' and thus rejected. Add numpy as a dev dependency, for testing it. As a consequence, the worker in test_crash_with_active_task now warns that numpy is installed but not imported, polluting the stderr under test; so import it. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/appose/shm.py | 21 +++++++++++++++++++++ tests/test_service.py | 4 +++- tests/test_shm.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2506796..989244e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ dev = [ "build", "mypy", + "numpy", "pytest", "ruff", "toml", diff --git a/src/appose/shm.py b/src/appose/shm.py index bb6dc79..50feb63 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -169,6 +169,27 @@ def ndarray(self): except ModuleNotFoundError: raise ImportError("NumPy is not available.") + @classmethod + def from_ndarray(cls, arr) -> NDArray: + """ + Create an NDArray in new shared memory, holding a copy of the given + NumPy array. + + The data is copied value by value, so the source array may be in any + byte order and memory layout (e.g. a big-endian array, or a transposed + view); the copy is always C-ordered, in native byte order. + + Args: + arr: The NumPy array to copy. + """ + nda = cls(arr.dtype.name, list(arr.shape)) + try: + nda.ndarray()[:] = arr + except BaseException: + nda.shm.dispose() + raise + return nda + def __enter__(self) -> Self: return self diff --git a/tests/test_service.py b/tests/test_service.py index d4e952f..c9029a0 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -247,7 +247,9 @@ def test_python_sys_exit(): def test_crash_with_active_task(): env = appose.system() - with env.python() as service: + # Note: Import numpy (a dev dependency) up front, so that the worker does + # not emit its numpy warning, which would pollute the stderr under test. + with env.python().init("import numpy") as service: maybe_debug(service) # Create a "long-running" task. script = ( diff --git a/tests/test_shm.py b/tests/test_shm.py index 9f628d8..166ae72 100644 --- a/tests/test_shm.py +++ b/tests/test_shm.py @@ -2,6 +2,7 @@ # Copyright (C) 2023 - 2026 Appose developers. # SPDX-License-Identifier: BSD-2-Clause +import numpy import pytest import appose @@ -125,3 +126,30 @@ def test_ndarray_normalizes_dtype(): with appose.NDArray("=u2", [3, 5]) as data: assert "uint16" == data.dtype assert 3 * 5 * 2 == data.shm.rsize + + +def test_from_ndarray_big_endian(): + # A big-endian array, as produced by some image readers. + src = (numpy.arange(3 * 4 * 5).reshape(3, 4, 5) * 1000).astype(">u2") + with pytest.raises(ValueError, match="native byte order"): + appose.NDArray(str(src.dtype), list(src.shape)) + with appose.NDArray.from_ndarray(src) as data: + assert "uint16" == data.dtype + assert [3, 4, 5] == data.shape + assert 3 * 4 * 5 * 2 == data.shm.rsize + dst = data.ndarray() + assert dst.dtype.isnative + assert numpy.array_equal(src, dst) + + +def test_from_ndarray_non_contiguous(): + src = numpy.arange(24, dtype="float32").reshape(2, 3, 4).transpose(2, 0, 1) + with appose.NDArray.from_ndarray(src) as data: + assert "float32" == data.dtype + assert [4, 2, 3] == data.shape + assert numpy.array_equal(src, data.ndarray()) + + +def test_from_ndarray_unsupported(): + with pytest.raises(ValueError, match="Unsupported dtype: datetime64"): + appose.NDArray.from_ndarray(numpy.zeros(3, dtype="datetime64[ns]")) From d59fade248c8a4797d5b037bc23eda779382db58 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 Sep 2026 15:48:19 -0500 Subject: [PATCH 6/7] Suggest the standard dtype name for explicit byte orders When a dtype like '>u2' is rejected, the error now names the standard dtype to use instead (e.g. 'uint16'), and points to arr.dtype.name and NDArray.from_ndarray. Also document that str(arr.dtype) is not suitable for non-native arrays, whereas arr.dtype.name is. Co-Authored-By: Claude Opus 5.5 --- src/appose/shm.py | 18 ++++++++++++------ tests/test_shm.py | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index 50feb63..c8c8038 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -131,7 +131,9 @@ def __init__(self, dtype: str, shape: list[int], shm: SharedMemory | None = None NumPy-style short forms (e.g. u2, f4, |u1, =c8) are also accepted, and normalized to the standard name (e.g. uint16, float32). Explicit byte orders (< or >) are rejected: Appose arrays - always use the machine's native byte order. + always use the machine's native byte order. To match a NumPy + array, pass arr.dtype.name, not str(arr.dtype), which keeps a + non-native byte order; or use from_ndarray to copy the array. shape: The dimensional extents; e.g. a stack of 7 image planes with resolution 512x512 would have shape [7, 512, 512]. shm: The SharedMemory containing the array data, or None to create it. @@ -265,11 +267,15 @@ def _normalize_dtype(dtype: str) -> str: if dtype in _DTYPE_SIZES: return dtype if dtype.startswith(("<", ">")): - raise ValueError( - f"Unsupported dtype: {dtype} " - "(Appose arrays are always native byte order; " - "omit the < or > prefix)" - ) + name = _DTYPE_ALIASES.get(dtype[1:], dtype[1:]) + if name in _DTYPE_SIZES: + raise ValueError( + f"Unsupported dtype: {dtype} " + "(Appose arrays are always in native byte order; " + f"use '{name}' instead, e.g. via arr.dtype.name, " + "or copy a NumPy array into shared memory " + "via NDArray.from_ndarray(arr))" + ) short = dtype[1:] if dtype.startswith(("=", "|")) else dtype if short not in _DTYPE_ALIASES: raise ValueError(f"Unsupported dtype: {dtype}") diff --git a/tests/test_shm.py b/tests/test_shm.py index 166ae72..b3975a0 100644 --- a/tests/test_shm.py +++ b/tests/test_shm.py @@ -90,8 +90,16 @@ def test_dtype_short_forms(): def test_dtype_explicit_byte_order(): - for dtype in ["u2", "c16", "float32"]: - with pytest.raises(ValueError, match="native byte order"): + for dtype, name in [ + ("u2", "uint16"), + ("c16", "complex128"), + ("float32", "float32"), + ]: + with pytest.raises(ValueError, match=f"native byte order; use '{name}'"): _normalize_dtype(dtype) @@ -114,6 +122,8 @@ def test_dtype_unsupported(): "float128", "intp", "U10", + "i3", "datetime64[ns]", "object", "FLOAT32", @@ -131,7 +141,7 @@ def test_ndarray_normalizes_dtype(): def test_from_ndarray_big_endian(): # A big-endian array, as produced by some image readers. src = (numpy.arange(3 * 4 * 5).reshape(3, 4, 5) * 1000).astype(">u2") - with pytest.raises(ValueError, match="native byte order"): + with pytest.raises(ValueError, match="use 'uint16'"): appose.NDArray(str(src.dtype), list(src.shape)) with appose.NDArray.from_ndarray(src) as data: assert "uint16" == data.dtype From d6a09b54542f1a4ab2c924c27f33b587f7049a1d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 25 Sep 2026 11:09:43 -0500 Subject: [PATCH 7/7] Rename NDArray.from_ndarray to copy_of, and deprecate ndarray() Name the NumPy-to-Appose copy after ShmImg.copyOf in imglib2-appose. For Appose-to-NumPy, implement __array__ so numpy.asarray(nda) wraps the shared memory without copying, and deprecate ndarray() in favor of it. Co-Authored-By: Claude Opus 5.5 --- src/appose/shm.py | 44 +++++++++++++++++++++++++++++++++----------- tests/test_shm.py | 41 +++++++++++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/appose/shm.py b/src/appose/shm.py index c8c8038..296dde3 100644 --- a/src/appose/shm.py +++ b/src/appose/shm.py @@ -8,6 +8,7 @@ from __future__ import annotations +import warnings from math import prod from multiprocessing import resource_tracker, shared_memory from typing import TYPE_CHECKING @@ -133,7 +134,7 @@ def __init__(self, dtype: str, shape: list[int], shm: SharedMemory | None = None Explicit byte orders (< or >) are rejected: Appose arrays always use the machine's native byte order. To match a NumPy array, pass arr.dtype.name, not str(arr.dtype), which keeps a - non-native byte order; or use from_ndarray to copy the array. + non-native byte order; or use copy_of to copy the array. shape: The dimensional extents; e.g. a stack of 7 image planes with resolution 512x512 would have shape [7, 512, 512]. shm: The SharedMemory containing the array data, or None to create it. @@ -156,23 +157,44 @@ def __str__(self): f"shm='{self.shm.name}' ({self.shm.rsize}))" ) - def ndarray(self): + def __array__(self, dtype=None, copy=None): """ - Create a NumPy ndarray object for working with the array data. - No array data is copied; the NumPy array wraps the same SharedMemory. + Support numpy.asarray(nda), which wraps the array data as a NumPy + ndarray without copying it; the NumPy array uses the same SharedMemory. Requires the numpy package to be installed. """ try: import numpy - - return numpy.ndarray( - prod(self.shape), dtype=self.dtype, buffer=self.shm.buf - ).reshape(self.shape) except ModuleNotFoundError: raise ImportError("NumPy is not available.") + arr = numpy.ndarray( + prod(self.shape), dtype=self.dtype, buffer=self.shm.buf + ).reshape(self.shape) + if dtype is None: + dtype = arr.dtype + if copy is False and numpy.dtype(dtype) != arr.dtype: + raise ValueError( + f"Cannot convert NDArray from {arr.dtype} to {dtype} without copying" + ) + return arr.astype(dtype, copy=bool(copy)) + + def ndarray(self): + """ + Create a NumPy ndarray object for working with the array data. + No array data is copied; the NumPy array wraps the same SharedMemory. + Requires the numpy package to be installed. + + Deprecated: use numpy.asarray(nda) instead. + """ + warnings.warn( + "NDArray.ndarray() is deprecated; use numpy.asarray(nda) instead", + DeprecationWarning, + stacklevel=2, + ) + return self.__array__() @classmethod - def from_ndarray(cls, arr) -> NDArray: + def copy_of(cls, arr) -> NDArray: """ Create an NDArray in new shared memory, holding a copy of the given NumPy array. @@ -186,7 +208,7 @@ def from_ndarray(cls, arr) -> NDArray: """ nda = cls(arr.dtype.name, list(arr.shape)) try: - nda.ndarray()[:] = arr + nda.__array__()[:] = arr except BaseException: nda.shm.dispose() raise @@ -274,7 +296,7 @@ def _normalize_dtype(dtype: str) -> str: "(Appose arrays are always in native byte order; " f"use '{name}' instead, e.g. via arr.dtype.name, " "or copy a NumPy array into shared memory " - "via NDArray.from_ndarray(arr))" + "via NDArray.copy_of(arr))" ) short = dtype[1:] if dtype.startswith(("=", "|")) else dtype if short not in _DTYPE_ALIASES: diff --git a/tests/test_shm.py b/tests/test_shm.py index b3975a0..29f34dc 100644 --- a/tests/test_shm.py +++ b/tests/test_shm.py @@ -138,28 +138,53 @@ def test_ndarray_normalizes_dtype(): assert 3 * 5 * 2 == data.shm.rsize -def test_from_ndarray_big_endian(): +def test_copy_of_big_endian(): # A big-endian array, as produced by some image readers. src = (numpy.arange(3 * 4 * 5).reshape(3, 4, 5) * 1000).astype(">u2") with pytest.raises(ValueError, match="use 'uint16'"): appose.NDArray(str(src.dtype), list(src.shape)) - with appose.NDArray.from_ndarray(src) as data: + with appose.NDArray.copy_of(src) as data: assert "uint16" == data.dtype assert [3, 4, 5] == data.shape assert 3 * 4 * 5 * 2 == data.shm.rsize - dst = data.ndarray() + dst = numpy.asarray(data) assert dst.dtype.isnative assert numpy.array_equal(src, dst) -def test_from_ndarray_non_contiguous(): +def test_copy_of_non_contiguous(): src = numpy.arange(24, dtype="float32").reshape(2, 3, 4).transpose(2, 0, 1) - with appose.NDArray.from_ndarray(src) as data: + with appose.NDArray.copy_of(src) as data: assert "float32" == data.dtype assert [4, 2, 3] == data.shape - assert numpy.array_equal(src, data.ndarray()) + assert numpy.array_equal(src, numpy.asarray(data)) -def test_from_ndarray_unsupported(): +def test_copy_of_unsupported(): with pytest.raises(ValueError, match="Unsupported dtype: datetime64"): - appose.NDArray.from_ndarray(numpy.zeros(3, dtype="datetime64[ns]")) + appose.NDArray.copy_of(numpy.zeros(3, dtype="datetime64[ns]")) + + +def test_asarray_zero_copy(): + with appose.NDArray("float32", [2, 3]) as data: + arr = numpy.asarray(data) + assert "float32" == arr.dtype.name + assert (2, 3) == arr.shape + arr[1, 2] = 42 + assert 42 == numpy.asarray(data)[1, 2] + + copied = numpy.array(data) + copied[0, 0] = 7 + assert 0 == numpy.asarray(data)[0, 0] + + converted = numpy.asarray(data, dtype="float64") + assert "float64" == converted.dtype.name + assert 42 == converted[1, 2] + + +def test_ndarray_deprecated(): + with appose.NDArray("uint8", [4]) as data: + with pytest.warns(DeprecationWarning, match="numpy.asarray"): + arr = data.ndarray() + arr[0] = 9 + assert 9 == numpy.asarray(data)[0]