Encode F-contiguous arrays in logical order

docs/release.md

@@ -8,6 +8,10 @@
 (unreleased)=
 
 ## Unreleased
+
+* Encode F-contiguous arrays in logical order for the vlen codecs and for delta, fixedscaleoffset, packbits, and categorize. They used to walk memory order, which did not match the logical layout.
+  By {user}`Corey Tasz <ceedot-rock>`, {issue}`850`, {pr}`863`
+
 -->
 
 (unreleased)=

src/numcodecs/categorize.py

@@ -53,8 +53,10 @@ def encode(self, buf):
         else:
             arr = ensure_ndarray(buf).view(self.dtype)
 
-        # flatten to simplify implementation
-        arr = arr.reshape(-1, order='A')
+        # flatten to simplify implementation, normalising to C order so the
+        # encoded stream holds elements in logical order regardless of the
+        # input's memory layout (see issue #850)
+        arr = np.ascontiguousarray(arr).reshape(-1)
 
         # setup output array
         enc = np.zeros_like(arr, dtype=self.astype)
@@ -79,8 +81,9 @@ def decode(self, buf, out=None):
         for i, label in enumerate(self.labels):
             dec[enc == (i + 1)] = label
 
-        # handle output
-        return ndarray_copy(dec, out)
+        # handle output; the decoded stream holds elements in logical (C)
+        # order, so copy them into the destination in C order (see #850)
+        return ndarray_copy(dec, out, order='C')
 
     def get_config(self):
         return {

src/numcodecs/compat.py

@@ -174,8 +174,22 @@ def ensure_text(s, encoding="utf-8"):
     return s
 
 
-def ndarray_copy(src, dst) -> NDArrayLike:
-    """Copy the contents of the array from `src` to `dst`."""
+def ndarray_copy(src, dst, order='A') -> NDArrayLike:
+    """Copy the contents of the array from `src` to `dst`.
+
+    Parameters
+    ----------
+    src : array-like
+        Source array, flattened before copying.
+    dst : array-like
+        Destination array.
+    order : {'A', 'C', 'F'}, optional
+        Memory layout used to interpret the flattened source data when
+        reshaping it to the destination's shape. The default 'A' preserves
+        the historical behavior of matching the destination's memory layout.
+        Pass 'C' when the source holds elements in logical (C) order,
+        regardless of the destination's memory layout (see issue #850).
+    """
 
     if dst is None:
         # no-op
@@ -194,10 +208,8 @@ def ndarray_copy(src, dst) -> NDArrayLike:
 
     # reshape source to match destination
     if src.shape != dst.shape:
-        if dst.flags.f_contiguous:
-            order = "F"
-        else:
-            order = "C"
+        if order == 'A':
+            order = 'F' if dst.flags.f_contiguous else 'C'
         src = src.reshape(dst.shape, order=order)
 
     # copy via numpy

src/numcodecs/delta.py

@@ -53,8 +53,10 @@ def encode(self, buf):
         # normalise input
         arr = ensure_ndarray(buf).view(self.dtype)
 
-        # flatten to simplify implementation
-        arr = arr.reshape(-1, order='A')
+        # flatten to simplify implementation, normalising to C order so the
+        # encoded stream holds elements in logical order regardless of the
+        # input's memory layout (see issue #850)
+        arr = np.ascontiguousarray(arr).reshape(-1)
 
         # setup encoded output
         enc = np.empty_like(arr, dtype=self.astype)
@@ -79,8 +81,9 @@ def decode(self, buf, out=None):
         # decode differences
         np.cumsum(enc, out=dec)
 
-        # handle output
-        return ndarray_copy(dec, out)
+        # handle output; the decoded stream holds elements in logical (C)
+        # order, so copy them into the destination in C order (see #850)
+        return ndarray_copy(dec, out, order='C')
 
     def get_config(self):
         # override to handle encoding dtypes

src/numcodecs/fixedscaleoffset.py

@@ -84,8 +84,10 @@ def encode(self, buf):
         # normalise input
         arr = ensure_ndarray(buf).view(self.dtype)
 
-        # flatten to simplify implementation
-        arr = arr.reshape(-1, order='A')
+        # flatten to simplify implementation, normalising to C order so the
+        # encoded stream holds elements in logical order regardless of the
+        # input's memory layout (see issue #850)
+        arr = np.ascontiguousarray(arr).reshape(-1)
 
         # compute scale offset
         enc = (arr - self.offset) * self.scale
@@ -109,8 +111,9 @@ def decode(self, buf, out=None):
         # convert dtype
         dec = dec.astype(self.dtype, copy=False)
 
-        # handle output
-        return ndarray_copy(dec, out)
+        # handle output; the decoded stream holds elements in logical (C)
+        # order, so copy them into the destination in C order (see #850)
+        return ndarray_copy(dec, out, order='C')
 
     def get_config(self):
         # override to handle encoding dtypes

src/numcodecs/packbits.py

@@ -33,8 +33,10 @@ def encode(self, buf):
         # normalise input
         arr = ensure_ndarray(buf).view(bool)
 
-        # flatten to simplify implementation
-        arr = arr.reshape(-1, order='A')
+        # flatten to simplify implementation, normalising to C order so the
+        # encoded stream holds elements in logical order regardless of the
+        # input's memory layout (see issue #850)
+        arr = np.ascontiguousarray(arr).reshape(-1)
 
         # determine size of packed data
         n = arr.size
@@ -78,5 +80,6 @@ def decode(self, buf, out=None):
         # view as boolean array
         dec = dec.view(bool)
 
-        # handle destination
-        return ndarray_copy(dec, out)
+        # handle destination; the decoded stream holds elements in logical
+        # (C) order, so copy them into the destination in C order (see #850)
+        return ndarray_copy(dec, out, order='C')

src/numcodecs/vlen.pyx

@@ -40,8 +40,7 @@ def check_out_param(out, n_items):
         raise TypeError('out must be 1-dimensional array')
     if out.dtype != object:
         raise ValueError('out must be object array')
-    out = out.reshape(-1, order='A')
-    if out.shape[0] < n_items:
+    if out.size < n_items:
         raise ValueError('out is too small')
     return out
 
@@ -87,8 +86,10 @@ class VLenUTF8(Codec):
             object o
             unicode u
 
-        # normalise input
-        input_values = np.asarray(buf, dtype=object).reshape(-1, order='A')
+        # normalise input to C order, so the encoded stream holds items
+        # in logical order regardless of the input's memory layout
+        # (see issue #850)
+        input_values = np.ascontiguousarray(buf, dtype=object).reshape(-1)
 
         # determine number of items
         n_items = input_values.shape[0]
@@ -154,11 +155,11 @@ class VLenUTF8(Codec):
         # load number of items
         n_items = load_le32(<uint8_t*>data)
 
-        # setup output
+        # setup output; always decode into a fresh 1-D array, which holds
+        # the items in logical (stream) order
         if out is not None:
             out = check_out_param(out, n_items)
-        else:
-            out = np.empty(n_items, dtype=object)
+        dec = np.empty(n_items, dtype=object)
 
         # iterate and decode - N.B., do not try to cast `out` as object[:]
         # as this causes segfaults, possibly similar to
@@ -171,10 +172,15 @@ class VLenUTF8(Codec):
             data += HEADER_LENGTH
             if data + L > data_end:
                 raise ValueError('corrupt buffer, data seem truncated')
-            out[i] = PyUnicode_FromStringAndSize(data, L)
+            dec[i] = PyUnicode_FromStringAndSize(data, L)
             data += L
 
-        return out
+        if out is not None:
+            # place the decoded items into the caller's buffer in logical
+            # (C) order (see issue #850)
+            out.flat[:n_items] = dec
+            return out
+        return dec
 
 
 class VLenBytes(Codec):
@@ -217,8 +223,10 @@ class VLenBytes(Codec):
             bytearray out
             char* data
 
-        # normalise input
-        values = np.asarray(buf, dtype=object).reshape(-1, order='A')
+        # normalise input to C order, so the encoded stream holds items
+        # in logical order regardless of the input's memory layout
+        # (see issue #850)
+        values = np.ascontiguousarray(buf, dtype=object).reshape(-1)
 
         # determine number of items
         n_items = values.shape[0]
@@ -283,11 +291,11 @@ class VLenBytes(Codec):
         # load number of items
         n_items = load_le32(<uint8_t*>data)
 
-        # setup output
+        # setup output; always decode into a fresh 1-D array, which holds
+        # the items in logical (stream) order
         if out is not None:
             out = check_out_param(out, n_items)
-        else:
-            out = np.empty(n_items, dtype=object)
+        dec = np.empty(n_items, dtype=object)
 
         # iterate and decode - N.B., do not try to cast `out` as object[:]
         # as this causes segfaults, possibly similar to
@@ -300,10 +308,15 @@ class VLenBytes(Codec):
             data += HEADER_LENGTH
             if data + L > data_end:
                 raise ValueError('corrupt buffer, data seem truncated')
-            out[i] = PyBytes_FromStringAndSize(data, L)
+            dec[i] = PyBytes_FromStringAndSize(data, L)
             data += L
 
-        return out
+        if out is not None:
+            # place the decoded items into the caller's buffer in logical
+            # (C) order (see issue #850)
+            out.flat[:n_items] = dec
+            return out
+        return dec
 
 
 class VLenArray(Codec):
@@ -361,8 +374,10 @@ class VLenArray(Codec):
             const Py_buffer* value_pb
             object o
 
-        # normalise input
-        values = np.asarray(buf, dtype=object).reshape(-1, order='A')
+        # normalise input to C order, so the encoded stream holds items
+        # in logical order regardless of the input's memory layout
+        # (see issue #850)
+        values = np.ascontiguousarray(buf, dtype=object).reshape(-1)
 
         # determine number of items
         n_items = values.shape[0]
@@ -439,11 +454,11 @@ class VLenArray(Codec):
         # load number of items
         n_items = load_le32(<uint8_t*>data)
 
-        # setup output
+        # setup output; always decode into a fresh 1-D array, which holds
+        # the items in logical (stream) order
         if out is not None:
             out = check_out_param(out, n_items)
-        else:
-            out = np.empty(n_items, dtype=object)
+        dec = np.empty(n_items, dtype=object)
 
         # iterate and decode - N.B., do not try to cast `out` as object[:]
         # as this causes segfaults, possibly similar to
@@ -463,7 +478,12 @@ class VLenArray(Codec):
             v_pb = PyMemoryView_GET_BUFFER(v_mv)
             memcpy(v_pb.buf, data, L)
 
-            out[i] = v
+            dec[i] = v
             data += L
 
-        return out
+        if out is not None:
+            # place the decoded items into the caller's buffer in logical
+            # (C) order (see issue #850)
+            out.flat[:n_items] = dec
+            return out
+        return dec