From 5c34a8d653ac534ff9b321be84d169e2029246ca Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 13:31:07 +0200 Subject: [PATCH 1/4] perf(blosc): use NEON SIMD for byte-shuffle on aarch64 c-blosc 1.x only provides SSE2/AVX2 shuffle kernels. On aarch64 its dispatcher always selects blosc_internal_(un)shuffle_generic, a scalar byte-by-byte loop that dominates Blosc runtime on ARM. On aarch64, compile a drop-in replacement for c-blosc's shuffle-generic.c that implements those two symbols with NEON kernels (copied from c-blosc2) for type sizes 2, 4, 8 and 16, falling back to the scalar loop otherwise. The c-blosc submodule is left untouched and the output is byte-identical. Controlled by a new `neon` meson option (default: auto). Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 10 + meson.options | 2 + .../_blosc_neon/shuffle-generic-neon.c | 372 ++++++++++++++++++ src/numcodecs/meson.build | 21 +- tests/test_blosc.py | 12 + 5 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 src/numcodecs/_blosc_neon/shuffle-generic-neon.c diff --git a/docs/release.md b/docs/release.md index 1f439a9c..fc1d3b59 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,16 @@ ## Unreleased +### Enhancements + +* Blosc byte-shuffle and unshuffle now use NEON SIMD on aarch64 (Apple Silicon, AWS + Graviton, etc). c-blosc 1.x only ships SSE2/AVX2 kernels, so on ARM it previously fell + back to a scalar loop that dominated Blosc runtime. The NEON kernels are taken from + c-blosc2 and produce byte-identical output. On an Apple M5, decoding 1 MiB lz4+shuffle + chunks is ~5-6x faster and encoding ~2x faster. Controlled by the new `neon` meson + option (default `auto`). + By {user}`Davis Bennett `. + (release_0.17.0)= * Allow building against a system Zlib. Note, this is only required if blosc is still diff --git a/meson.options b/meson.options index f055af87..63378bb0 100644 --- a/meson.options +++ b/meson.options @@ -10,3 +10,5 @@ option('sse2', type: 'feature', value: 'auto', description: 'Enable SSE2 SIMD optimizations') option('avx2', type: 'feature', value: 'auto', description: 'Enable AVX2 SIMD optimizations') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON SIMD shuffle on aarch64') diff --git a/src/numcodecs/_blosc_neon/shuffle-generic-neon.c b/src/numcodecs/_blosc_neon/shuffle-generic-neon.c new file mode 100644 index 00000000..a7e46306 --- /dev/null +++ b/src/numcodecs/_blosc_neon/shuffle-generic-neon.c @@ -0,0 +1,372 @@ +/********************************************************************* + NEON-accelerated drop-in replacement for c-blosc's shuffle-generic.c. + + c-blosc 1.x has no ARM SIMD shuffle, so on aarch64 its dispatcher always + selects blosc_internal_(un)shuffle_generic, a scalar byte-by-byte loop. + This file defines those two symbols with NEON kernels for type sizes + 2, 4, 8 and 16, falling back to the scalar loop for other type sizes and + for the tail of each block. The byte layout is identical to the generic + routines, so the Blosc format is unchanged. + + The kernels below (shuffleN_neon / unshuffleN_neon) are copied verbatim + from c-blosc2 (blosc/shuffle-neon.c): + + Copyright (c) 2021 Lucian Marc + https://blosc.org + License: BSD 3-Clause (see c-blosc/LICENSE.txt) +**********************************************************************/ + +#include "shuffle-generic.h" + +#if !defined(__ARM_NEON) && !defined(_M_ARM64) +#error "shuffle-generic-neon.c requires a NEON-capable target" +#endif + +#include +#include + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 2; + uint8x16x2_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 32, k++) { + /* Load (and permute) 32 bytes to the structure r0 */ + r0 = vld2q_u8(src + i); + /* Store the results in the destination vector */ + vst1q_u8(dest + total_elements * 0 + k * 16, r0.val[0]); + vst1q_u8(dest + total_elements * 1 + k * 16, r0.val[1]); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 4; + uint8x16x4_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load (and permute) 64 bytes to the structure r0 */ + r0 = vld4q_u8(src + i); + /* Store the results in the destination vector */ + vst1q_u8(dest + total_elements * 0 + k * 16, r0.val[0]); + vst1q_u8(dest + total_elements * 1 + k * 16, r0.val[1]); + vst1q_u8(dest + total_elements * 2 + k * 16, r0.val[2]); + vst1q_u8(dest + total_elements * 3 + k * 16, r0.val[3]); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 8; + uint8x8x2_t r0[4]; + uint16x4x2_t r1[4]; + uint32x2x2_t r2[4]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load and interleave groups of 8 bytes (64 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 1 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 2 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 5 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 6 * 8), vld1_u8(src + i + 7 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + k * 8 + 0 * total_elements, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + k * 8 + 1 * total_elements, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + k * 8 + 2 * total_elements, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + k * 8 + 3 * total_elements, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + k * 8 + 4 * total_elements, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + k * 8 + 5 * total_elements, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + k * 8 + 6 * total_elements, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + k * 8 + 7 * total_elements, vreinterpret_u8_u32(r2[3].val[1])); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 16; + uint8x8x2_t r0[8]; + uint16x4x2_t r1[8]; + uint32x2x2_t r2[8]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 128, k++) { + /* Load and interleave groups of 16 bytes (128 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 2 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 1 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 6 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 5 * 8), vld1_u8(src + i + 7 * 8)); + r0[4] = vzip_u8(vld1_u8(src + i + 8 * 8), vld1_u8(src + i + 10 * 8)); + r0[5] = vzip_u8(vld1_u8(src + i + 9 * 8), vld1_u8(src + i + 11 * 8)); + r0[6] = vzip_u8(vld1_u8(src + i + 12 * 8), vld1_u8(src + i + 14 * 8)); + r0[7] = vzip_u8(vld1_u8(src + i + 13 * 8), vld1_u8(src + i + 15 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[2].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[2].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[1].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[1].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + r1[4] = vzip_u16(vreinterpret_u16_u8(r0[4].val[0]), vreinterpret_u16_u8(r0[6].val[0])); + r1[5] = vzip_u16(vreinterpret_u16_u8(r0[4].val[1]), vreinterpret_u16_u8(r0[6].val[1])); + r1[6] = vzip_u16(vreinterpret_u16_u8(r0[5].val[0]), vreinterpret_u16_u8(r0[7].val[0])); + r1[7] = vzip_u16(vreinterpret_u16_u8(r0[5].val[1]), vreinterpret_u16_u8(r0[7].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[4].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[4].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[5].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[5].val[1])); + r2[4] = vzip_u32(vreinterpret_u32_u16(r1[2].val[0]), vreinterpret_u32_u16(r1[6].val[0])); + r2[5] = vzip_u32(vreinterpret_u32_u16(r1[2].val[1]), vreinterpret_u32_u16(r1[6].val[1])); + r2[6] = vzip_u32(vreinterpret_u32_u16(r1[3].val[0]), vreinterpret_u32_u16(r1[7].val[0])); + r2[7] = vzip_u32(vreinterpret_u32_u16(r1[3].val[1]), vreinterpret_u32_u16(r1[7].val[1])); + /* Store the results to the destination vector */ + vst1_u8(dest + k * 8 + 0 * total_elements, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + k * 8 + 1 * total_elements, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + k * 8 + 2 * total_elements, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + k * 8 + 3 * total_elements, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + k * 8 + 4 * total_elements, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + k * 8 + 5 * total_elements, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + k * 8 + 6 * total_elements, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + k * 8 + 7 * total_elements, vreinterpret_u8_u32(r2[3].val[1])); + vst1_u8(dest + k * 8 + 8 * total_elements, vreinterpret_u8_u32(r2[4].val[0])); + vst1_u8(dest + k * 8 + 9 * total_elements, vreinterpret_u8_u32(r2[4].val[1])); + vst1_u8(dest + k * 8 + 10 * total_elements, vreinterpret_u8_u32(r2[5].val[0])); + vst1_u8(dest + k * 8 + 11 * total_elements, vreinterpret_u8_u32(r2[5].val[1])); + vst1_u8(dest + k * 8 + 12 * total_elements, vreinterpret_u8_u32(r2[6].val[0])); + vst1_u8(dest + k * 8 + 13 * total_elements, vreinterpret_u8_u32(r2[6].val[1])); + vst1_u8(dest + k * 8 + 14 * total_elements, vreinterpret_u8_u32(r2[7].val[0])); + vst1_u8(dest + k * 8 + 15 * total_elements, vreinterpret_u8_u32(r2[7].val[1])); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 2; + uint8x16x2_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 32, k++) { + /* Load 32 bytes to the structure r0 */ + r0.val[0] = vld1q_u8(src + total_elements * 0 + k * 16); + r0.val[1] = vld1q_u8(src + total_elements * 1 + k * 16); + /* Store (with permutation) the results in the destination vector */ + vst2q_u8(dest + k * 32, r0); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 4; + uint8x16x4_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* load 64 bytes to the structure r0 */ + r0.val[0] = vld1q_u8(src + total_elements * 0 + k * 16); + r0.val[1] = vld1q_u8(src + total_elements * 1 + k * 16); + r0.val[2] = vld1q_u8(src + total_elements * 2 + k * 16); + r0.val[3] = vld1q_u8(src + total_elements * 3 + k * 16); + /* Store (with permutation) the results in the destination vector */ + vst4q_u8(dest + k * 64, r0); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 8; + uint8x8x2_t r0[4]; + uint16x4x2_t r1[4]; + uint32x2x2_t r2[4]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load and interleave groups of 8 bytes (64 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + 0 * total_elements + k * 8), vld1_u8(src + 1 * total_elements + k * 8)); + r0[1] = vzip_u8(vld1_u8(src + 2 * total_elements + k * 8), vld1_u8(src + 3 * total_elements + k * 8)); + r0[2] = vzip_u8(vld1_u8(src + 4 * total_elements + k * 8), vld1_u8(src + 5 * total_elements + k * 8)); + r0[3] = vzip_u8(vld1_u8(src + 6 * total_elements + k * 8), vld1_u8(src + 7 * total_elements + k * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + i + 0 * 8, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + i + 1 * 8, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + i + 2 * 8, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + i + 3 * 8, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + i + 4 * 8, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + i + 5 * 8, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + i + 6 * 8, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + i + 7 * 8, vreinterpret_u8_u32(r2[3].val[1])); + + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 16; + uint8x8x2_t r0[8]; + uint16x4x2_t r1[8]; + uint32x2x2_t r2[8]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 128, k++) { + /* Load and interleave groups of 16 bytes (128 bytes) to the structure r0*/ + r0[0] = vzip_u8(vld1_u8(src + k * 8 + 0 * total_elements), vld1_u8(src + k * 8 + 1 * total_elements)); + r0[1] = vzip_u8(vld1_u8(src + k * 8 + 2 * total_elements), vld1_u8(src + k * 8 + 3 * total_elements)); + r0[2] = vzip_u8(vld1_u8(src + k * 8 + 4 * total_elements), vld1_u8(src + k * 8 + 5 * total_elements)); + r0[3] = vzip_u8(vld1_u8(src + k * 8 + 6 * total_elements), vld1_u8(src + k * 8 + 7 * total_elements)); + r0[4] = vzip_u8(vld1_u8(src + k * 8 + 8 * total_elements), vld1_u8(src + k * 8 + 9 * total_elements)); + r0[5] = vzip_u8(vld1_u8(src + k * 8 + 10 * total_elements), vld1_u8(src + k * 8 + 11 * total_elements)); + r0[6] = vzip_u8(vld1_u8(src + k * 8 + 12 * total_elements), vld1_u8(src + k * 8 + 13 * total_elements)); + r0[7] = vzip_u8(vld1_u8(src + k * 8 + 14 * total_elements), vld1_u8(src + k * 8 + 15 * total_elements)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + r1[4] = vzip_u16(vreinterpret_u16_u8(r0[4].val[0]), vreinterpret_u16_u8(r0[5].val[0])); + r1[5] = vzip_u16(vreinterpret_u16_u8(r0[4].val[1]), vreinterpret_u16_u8(r0[5].val[1])); + r1[6] = vzip_u16(vreinterpret_u16_u8(r0[6].val[0]), vreinterpret_u16_u8(r0[7].val[0])); + r1[7] = vzip_u16(vreinterpret_u16_u8(r0[6].val[1]), vreinterpret_u16_u8(r0[7].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + r2[4] = vzip_u32(vreinterpret_u32_u16(r1[4].val[0]), vreinterpret_u32_u16(r1[6].val[0])); + r2[5] = vzip_u32(vreinterpret_u32_u16(r1[4].val[1]), vreinterpret_u32_u16(r1[6].val[1])); + r2[6] = vzip_u32(vreinterpret_u32_u16(r1[5].val[0]), vreinterpret_u32_u16(r1[7].val[0])); + r2[7] = vzip_u32(vreinterpret_u32_u16(r1[5].val[1]), vreinterpret_u32_u16(r1[7].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + i + 0 * 8, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + i + 1 * 8, vreinterpret_u8_u32(r2[4].val[0])); + vst1_u8(dest + i + 2 * 8, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + i + 3 * 8, vreinterpret_u8_u32(r2[4].val[1])); + vst1_u8(dest + i + 4 * 8, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + i + 5 * 8, vreinterpret_u8_u32(r2[5].val[0])); + vst1_u8(dest + i + 6 * 8, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + i + 7 * 8, vreinterpret_u8_u32(r2[5].val[1])); + vst1_u8(dest + i + 8 * 8, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + i + 9 * 8, vreinterpret_u8_u32(r2[6].val[0])); + vst1_u8(dest + i + 10 * 8, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + i + 11 * 8, vreinterpret_u8_u32(r2[6].val[1])); + vst1_u8(dest + i + 12 * 8, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + i + 13 * 8, vreinterpret_u8_u32(r2[7].val[0])); + vst1_u8(dest + i + 14 * 8, vreinterpret_u8_u32(r2[3].val[1])); + vst1_u8(dest + i + 15 * 8, vreinterpret_u8_u32(r2[7].val[1])); + } +} + +/* Size in bytes of the chunk processed by one iteration of a NEON kernel, + or 0 if there is no NEON kernel for this type size. */ +static size_t neon_chunk_size(const size_t bytesoftype) { + switch (bytesoftype) { + case 2: + case 4: + return bytesoftype * 16; + case 8: + case 16: + return bytesoftype * 8; + default: + return 0; + } +} + +/* Shuffle a block. This can never fail. */ +void blosc_internal_shuffle_generic(const size_t bytesoftype, const size_t blocksize, + const uint8_t* const _src, uint8_t* const _dest) +{ + const size_t chunk = neon_chunk_size(bytesoftype); + size_t vectorizable_bytes, vectorizable_elements, total_elements; + + if (chunk == 0 || blocksize < chunk) { + shuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); + return; + } + vectorizable_bytes = blocksize - (blocksize % chunk); + vectorizable_elements = vectorizable_bytes / bytesoftype; + total_elements = blocksize / bytesoftype; + + switch (bytesoftype) { + case 2: + shuffle2_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_neon(_dest, _src, vectorizable_elements, total_elements); + break; + } + /* Handles the non-vectorizable tail and copies any leftover bytes. */ + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); +} + +/* Unshuffle a block. This can never fail. */ +void blosc_internal_unshuffle_generic(const size_t bytesoftype, const size_t blocksize, + const uint8_t* const _src, uint8_t* const _dest) +{ + const size_t chunk = neon_chunk_size(bytesoftype); + size_t vectorizable_bytes, vectorizable_elements, total_elements; + + if (chunk == 0 || blocksize < chunk) { + unshuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); + return; + } + vectorizable_bytes = blocksize - (blocksize % chunk); + vectorizable_elements = vectorizable_bytes / bytesoftype; + total_elements = blocksize / bytesoftype; + + switch (bytesoftype) { + case 2: + unshuffle2_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_neon(_dest, _src, vectorizable_elements, total_elements); + break; + } + /* Handles the non-vectorizable tail and copies any leftover bytes. */ + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); +} diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 41a3a7a2..ed813f01 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -12,6 +12,7 @@ endif # Correct for cross-compilation: checks what the *target* supports. have_sse2 = false have_avx2 = false +have_neon = false if host_machine.cpu_family() == 'x86_64' # Honor legacy environment variables for backwards compatibility with @@ -38,6 +39,17 @@ if host_machine.cpu_family() == 'x86_64' endif endif +# NEON is mandatory on aarch64, so no runtime detection is needed. +if host_machine.cpu_family() == 'aarch64' and not get_option('neon').disabled() + have_neon = cc.compiles(''' + #if !defined(__ARM_NEON) && !defined(_M_ARM64) + #error "no NEON" + #endif + #include + int main(void) { uint8x16_t v = vdupq_n_u8(0); return vgetq_lane_u8(v, 0); } + ''', name: 'NEON intrinsics') +endif + # --- Vendored zstd --- zstd_dep = dependency('libzstd', required: get_option('system_zstd')) @@ -165,10 +177,17 @@ if not blosc_dep.found() '../../c-blosc/blosc/blosclz.c', '../../c-blosc/blosc/fastcopy.c', '../../c-blosc/blosc/shuffle.c', - '../../c-blosc/blosc/shuffle-generic.c', '../../c-blosc/blosc/bitshuffle-generic.c', ) + # c-blosc 1.x has no ARM SIMD shuffle; on aarch64 swap in a NEON + # implementation of the generic (un)shuffle routines. + if have_neon + blosc_sources += files('_blosc_neon/shuffle-generic-neon.c') + else + blosc_sources += files('../../c-blosc/blosc/shuffle-generic.c') + endif + blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] if have_sse2 diff --git a/tests/test_blosc.py b/tests/test_blosc.py index b920f03e..b54efcf6 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -330,3 +330,15 @@ def test_config_no_typesize(): codec = Blosc(shuffle=Blosc.SHUFFLE, typesize=5) config = codec.get_config() assert "typesize" not in config + + +@pytest.mark.parametrize('shuffle', [Blosc.SHUFFLE, Blosc.BITSHUFFLE]) +@pytest.mark.parametrize('blocksize', [0, 2**8, 4097]) +@pytest.mark.parametrize('nbytes', [1, 31, 33, 127, 1000, 65537]) +@pytest.mark.parametrize('typesize', [1, 2, 3, 4, 7, 8, 12, 16, 17, 32]) +def test_shuffle_roundtrip_typesizes(typesize, nbytes, blocksize, shuffle): + # Exercises the SIMD (un)shuffle kernels, including the scalar tail used when a + # block is not a multiple of the vector width or the typesize. + src = np.random.default_rng(0).integers(0, 64, nbytes, dtype='u1').tobytes() + codec = Blosc(cname='lz4', clevel=5, shuffle=shuffle, blocksize=blocksize, typesize=typesize) + assert bytes(codec.decode(codec.encode(src))) == src From a280de3db5621fbb6aad4d88c30876542251727a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 13:41:33 +0200 Subject: [PATCH 2/4] build: replace c-blosc git submodule with meson wrap subprojects Replace the c-blosc submodule and the hand-maintained source lists for its internal-complibs with meson subprojects: - zstd, lz4 and zlib come from WrapDB (subprojects/*.wrap), so their versions are no longer tied to c-blosc's bundled copies. - c-blosc 1.x is not in WrapDB, so subprojects/c-blosc.wrap pins the same commit as the old submodule, and subprojects/packagefiles/c-blosc/ provides its meson.build plus numcodecs patches (the NEON shuffle). - The system_* options keep their meaning; when a library is not taken from the system, the subproject is configured first so every lookup, including c-blosc's, resolves to the same static build. - meson-python skips installing subproject artifacts and includes the subproject sources in the sdist, so the sdist builds offline. - License texts for the bundled libraries now live in LICENSES/, adding the previously missing zstd license. Assisted-by: ClaudeCode:claude-opus-5-5 --- .gitignore | 6 +- .gitmodules | 3 - LICENSES/BITSHUFFLE.txt | 21 ++ LICENSES/BLOSC.txt | 31 +++ LICENSES/FASTLZ.txt | 20 ++ LICENSES/LZ4.txt | 24 ++ LICENSES/ZLIB.txt | 22 ++ LICENSES/ZSTD.txt | 30 +++ c-blosc | 1 - docs/contributing.md | 15 +- docs/index.md | 5 +- pyproject.toml | 10 +- src/numcodecs/meson.build | 249 ++++-------------- subprojects/c-blosc.wrap | 15 ++ subprojects/lz4.wrap | 13 + .../c-blosc/blosc}/shuffle-generic-neon.c | 0 subprojects/packagefiles/c-blosc/meson.build | 88 +++++++ .../packagefiles/c-blosc/meson.options | 6 + subprojects/zlib.wrap | 14 + subprojects/zstd.wrap | 13 + 20 files changed, 363 insertions(+), 223 deletions(-) delete mode 100644 .gitmodules create mode 100644 LICENSES/BITSHUFFLE.txt create mode 100644 LICENSES/BLOSC.txt create mode 100644 LICENSES/FASTLZ.txt create mode 100644 LICENSES/LZ4.txt create mode 100644 LICENSES/ZLIB.txt create mode 100644 LICENSES/ZSTD.txt delete mode 160000 c-blosc create mode 100644 subprojects/c-blosc.wrap create mode 100644 subprojects/lz4.wrap rename {src/numcodecs/_blosc_neon => subprojects/packagefiles/c-blosc/blosc}/shuffle-generic-neon.c (100%) create mode 100644 subprojects/packagefiles/c-blosc/meson.build create mode 100644 subprojects/packagefiles/c-blosc/meson.options create mode 100644 subprojects/zlib.wrap create mode 100644 subprojects/zstd.wrap diff --git a/.gitignore b/.gitignore index 14facfe7..e04e23db 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,8 @@ ENV/ .pixi/* *.egg-info pixi.lock -uv.lock \ No newline at end of file +uv.lock +# meson subprojects: only wraps and our overlays are tracked +/subprojects/* +!/subprojects/*.wrap +!/subprojects/packagefiles/ diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 04c71aa1..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "c-blosc"] - path = c-blosc - url = https://github.com/Blosc/c-blosc.git diff --git a/LICENSES/BITSHUFFLE.txt b/LICENSES/BITSHUFFLE.txt new file mode 100644 index 00000000..1365ed69 --- /dev/null +++ b/LICENSES/BITSHUFFLE.txt @@ -0,0 +1,21 @@ +Bitshuffle - Filter for improving compression of typed binary data. + +Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/LICENSES/BLOSC.txt b/LICENSES/BLOSC.txt new file mode 100644 index 00000000..e70b69f3 --- /dev/null +++ b/LICENSES/BLOSC.txt @@ -0,0 +1,31 @@ +BSD License + +For Blosc - A blocking, shuffling and lossless compression library + +Copyright (c) 2009-2018 Francesc Alted +Copyright (c) 2019-present Blosc Development Team + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Francesc Alted nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSES/FASTLZ.txt b/LICENSES/FASTLZ.txt new file mode 100644 index 00000000..b22705f9 --- /dev/null +++ b/LICENSES/FASTLZ.txt @@ -0,0 +1,20 @@ +FastLZ - Byte-aligned LZ77 compression library +Copyright (C) 2005-2020 Ariya Hidayat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/LICENSES/LZ4.txt b/LICENSES/LZ4.txt new file mode 100644 index 00000000..48849169 --- /dev/null +++ b/LICENSES/LZ4.txt @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSES/ZLIB.txt b/LICENSES/ZLIB.txt new file mode 100644 index 00000000..b7a69d05 --- /dev/null +++ b/LICENSES/ZLIB.txt @@ -0,0 +1,22 @@ +Copyright notice: + + (C) 1995-2026 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu diff --git a/LICENSES/ZSTD.txt b/LICENSES/ZSTD.txt new file mode 100644 index 00000000..75800288 --- /dev/null +++ b/LICENSES/ZSTD.txt @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/c-blosc b/c-blosc deleted file mode 160000 index 04c06fb7..00000000 --- a/c-blosc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 04c06fb7339d3df3600b1862077b8699b30a0e1d diff --git a/docs/contributing.md b/docs/contributing.md index e4bd9b73..8bed6954 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -62,13 +62,13 @@ You will need your own fork to work on the code. Go to the link above and hit the "Fork" button. Then clone your fork to your local machine: ``` -$ git clone --recursive git@github.com:your-user-name/numcodecs.git # with ssh +$ git clone git@github.com:your-user-name/numcodecs.git # with ssh ``` or: ``` -$ git clone --recursive https://github.com/your-user-name/numcodecs.git # with https +$ git clone https://github.com/your-user-name/numcodecs.git # with https ``` Then `cd` into the clone and add the `upstream` remote: @@ -78,12 +78,11 @@ $ cd numcodecs $ git remote add upstream https://github.com/zarr-developers/numcodecs.git ``` -Note the ``--recursive`` flag is required to clone the ``c-blosc`` git submodule. If you -forgot it, you can initialize it later with: - -``` -$ git submodule update --init --recursive -``` +The C libraries numcodecs links against (c-blosc, zstd, lz4, zlib) are not stored in the +repository. They are pinned in `subprojects/*.wrap` and downloaded by meson the first time +you build, so the first build needs network access. To fetch them ahead of time, run +`meson subprojects download`. Numcodecs-specific build files and patches for c-blosc live in +`subprojects/packagefiles/c-blosc/`. ### Creating a development environment diff --git a/docs/index.md b/docs/index.md index bdaf1c8c..3ed83836 100644 --- a/docs/index.md +++ b/docs/index.md @@ -58,14 +58,11 @@ To work with Numcodecs source code in development, see the development environment with venv or uv. ``` -$ git clone --recursive https://github.com/zarr-developers/numcodecs.git +$ git clone https://github.com/zarr-developers/numcodecs.git $ cd numcodecs $ pip install -e .[test,msgpack,zfpy] ``` -Note: if you prefer to use the GitHub CLI `gh` you will need to append `-- --recurse-submodules` -to the clone command to everything works properly. - To verify that Numcodecs has been fully installed (including the Blosc extension) run the test suite: diff --git a/pyproject.toml b/pyproject.toml index 185ecc3a..41eb9a72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,8 +36,7 @@ maintainers = [ license = "MIT" license-files = [ "LICENSE.txt", - "c-blosc/LICENSE.txt", - "c-blosc/LICENSES/*", + "LICENSES/*", ] [project.urls] @@ -238,6 +237,13 @@ warn_redundant_casts = true warn_unused_ignores = true warn_unused_configs = true +[tool.meson-python.args] +# C libraries are built as static subprojects and linked into the extension +# modules; don't install their headers, static libraries or pkg-config files. +install = ["--skip-subprojects"] +# Ship the subproject sources in the sdist so it builds without network access. +dist = ["--include-subprojects"] + [tool.uv] # meson-python editable installs require build deps (ninja, meson) at runtime for # auto-rebuild on import. Skip build isolation so the venv's ninja is used. diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index ed813f01..f7eab5cb 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -7,222 +7,63 @@ if host_machine.system() != 'windows' link_args += ['-pthread'] endif -# --- SIMD detection --- -# Detected via compiler capability, NOT runtime CPU flags. -# Correct for cross-compilation: checks what the *target* supports. -have_sse2 = false -have_avx2 = false -have_neon = false - -if host_machine.cpu_family() == 'x86_64' - # Honor legacy environment variables for backwards compatibility with - # existing build scripts and packaging recipes. - disable_sse2_env = run_command( - py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_SSE2", ""))', - check: true, - ).stdout().strip() != '' - disable_avx2_env = run_command( - py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_AVX2", ""))', - check: true, - ).stdout().strip() != '' - - if get_option('sse2').disabled() or disable_sse2_env - have_sse2 = false - else - have_sse2 = cc.has_argument('-msse2') - endif - - if get_option('avx2').disabled() or disable_avx2_env - have_avx2 = false - else - have_avx2 = cc.has_argument('-mavx2') - endif -endif - -# NEON is mandatory on aarch64, so no runtime detection is needed. -if host_machine.cpu_family() == 'aarch64' and not get_option('neon').disabled() - have_neon = cc.compiles(''' - #if !defined(__ARM_NEON) && !defined(_M_ARM64) - #error "no NEON" - #endif - #include - int main(void) { uint8x16_t v = vdupq_n_u8(0); return vgetq_lane_u8(v, 0); } - ''', name: 'NEON intrinsics') -endif - -# --- Vendored zstd --- -zstd_dep = dependency('libzstd', required: get_option('system_zstd')) - +# --- C library dependencies --- +# Each codec library is either found on the system (-Dsystem_=enabled/auto) +# or built as a static subproject from subprojects/.wrap. Once a subproject +# has been configured, every later dependency('') lookup -- including those +# inside the c-blosc subproject -- resolves to it, so numcodecs.zstd and blosc +# always link the same zstd. +subproject_opts = ['default_library=static', 'warning_level=0'] + +zstd_dep = dependency('libzstd', required: get_option('system_zstd'), allow_fallback: false) if not zstd_dep.found() - zstd_sources = files( - # common/ - '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', - # compress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', - # decompress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', - # dictBuilder/ - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', - ) - - # Assembly files — meson handles .S natively - if host_machine.cpu_family() == 'x86_64' - zstd_sources += files( - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', - ) - endif - - zstd_inc = include_directories( - '../../c-blosc/internal-complibs/zstd-1.5.6', - '../../c-blosc/internal-complibs/zstd-1.5.6/common', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', - ) - - zstd_lib = static_library('zstd_vendored', zstd_sources, - include_directories: zstd_inc, - c_args: c_args, - ) - zstd_dep = declare_dependency( - link_with: zstd_lib, - include_directories: zstd_inc, - ) + subproject('zstd', default_options: subproject_opts + [ + 'bin_programs=false', 'multi_thread=disabled', 'legacy_level=0', + 'zlib=disabled', 'lzma=disabled', 'lz4=disabled', + ]) + zstd_dep = dependency('libzstd') endif -# --- Vendored lz4 --- -lz4_dep = dependency('liblz4', required: get_option('system_lz4')) - +lz4_dep = dependency('liblz4', required: get_option('system_lz4'), allow_fallback: false) if not lz4_dep.found() - lz4_sources = files( - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', - ) - lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') - - lz4_lib = static_library('lz4_vendored', lz4_sources, - include_directories: lz4_inc, - c_args: c_args, - ) - lz4_dep = declare_dependency( - link_with: lz4_lib, - include_directories: lz4_inc, - ) + subproject('lz4', default_options: subproject_opts + ['ossfuzz=false', 'programs=false']) + lz4_dep = dependency('liblz4') endif -# --- Vendored blosc --- -blosc_dep = dependency('blosc', required: get_option('system_blosc')) - +blosc_dep = dependency('blosc', required: get_option('system_blosc'), allow_fallback: false) if not blosc_dep.found() - # --- Vendored zlib --- - zlib_dep = dependency('zlib', required: get_option('system_zlib')) - + # zlib is only needed by blosc; it is not built when using a system blosc. + zlib_dep = dependency('zlib', required: get_option('system_zlib'), allow_fallback: false) if not zlib_dep.found() - zlib_sources = files( - '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', - ) - zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') - - zlib_lib = static_library('zlib_vendored', zlib_sources, - include_directories: zlib_inc, - c_args: c_args, - ) - zlib_dep = declare_dependency( - link_with: zlib_lib, - include_directories: zlib_inc, - ) - endif - - blosc_sources = files( - '../../c-blosc/blosc/blosc.c', - '../../c-blosc/blosc/blosclz.c', - '../../c-blosc/blosc/fastcopy.c', - '../../c-blosc/blosc/shuffle.c', - '../../c-blosc/blosc/bitshuffle-generic.c', - ) - - # c-blosc 1.x has no ARM SIMD shuffle; on aarch64 swap in a NEON - # implementation of the generic (un)shuffle routines. - if have_neon - blosc_sources += files('_blosc_neon/shuffle-generic-neon.c') - else - blosc_sources += files('../../c-blosc/blosc/shuffle-generic.c') - endif - - blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] - - if have_sse2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-sse2.c', - '../../c-blosc/blosc/bitshuffle-sse2.c', - ) - blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__SSE2__'] - endif + subproject('zlib', default_options: subproject_opts) endif - if have_avx2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-avx2.c', - '../../c-blosc/blosc/bitshuffle-avx2.c', - ) - blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__AVX2__'] + # Honor legacy environment variables for backwards compatibility with + # existing build scripts and packaging recipes. + env_disabled = {} + foreach simd : ['sse2', 'avx2'] + env_disabled += {simd: run_command( + py, '-c', + 'import os; print(os.environ.get("DISABLE_NUMCODECS_@0@", ""))'.format(simd.to_upper()), + check: true, + ).stdout().strip() != ''} + endforeach + + simd_opts = [] + foreach simd : ['sse2', 'avx2', 'neon'] + f = get_option(simd) + if f.disabled() or env_disabled.get(simd, false) + state = 'disabled' + elif f.enabled() + state = 'enabled' + else + state = 'auto' endif - endif - - blosc_inc = include_directories('../../c-blosc/blosc') + simd_opts += '@0@=@1@'.format(simd, state) + endforeach - blosc_lib = static_library('blosc_vendored', blosc_sources, - include_directories: blosc_inc, - dependencies: [lz4_dep, zlib_dep, zstd_dep], - c_args: blosc_c_args, - ) - blosc_dep = declare_dependency( - link_with: blosc_lib, - include_directories: blosc_inc, - ) + subproject('c-blosc', default_options: subproject_opts + simd_opts) + blosc_dep = dependency('blosc') endif # --- Cython extension modules --- diff --git a/subprojects/c-blosc.wrap b/subprojects/c-blosc.wrap new file mode 100644 index 00000000..ba5d22d0 --- /dev/null +++ b/subprojects/c-blosc.wrap @@ -0,0 +1,15 @@ +[wrap-file] +# c-blosc 1.x is not in WrapDB, so this wrap is maintained here. It pins the +# c-blosc commit numcodecs was previously pinned to as a git submodule (main +# after v1.21.6, needed for gcc 15). Switch to a release tarball once c-blosc +# tags a release that includes it. +directory = c-blosc-04c06fb7339d3df3600b1862077b8699b30a0e1d +source_url = https://github.com/Blosc/c-blosc/archive/04c06fb7339d3df3600b1862077b8699b30a0e1d.tar.gz +source_filename = c-blosc-04c06fb7339d3df3600b1862077b8699b30a0e1d.tar.gz +source_hash = 22c1d0fa6500f319db9a9da5e9344e3a2f26de98e3beadf25f96dd3c71a271f3 +# meson.build/meson.options for c-blosc (upstream only ships CMake), plus +# numcodecs-specific sources such as the NEON shuffle. +patch_directory = c-blosc + +[provide] +blosc = blosc_dep diff --git a/subprojects/lz4.wrap b/subprojects/lz4.wrap new file mode 100644 index 00000000..7732b9a2 --- /dev/null +++ b/subprojects/lz4.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = lz4-1.10.0 +source_url = https://github.com/lz4/lz4/archive/v1.10.0.tar.gz +source_filename = lz4-1.10.0.tgz +source_hash = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b +patch_filename = lz4_1.10.0-2_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/lz4_1.10.0-2/get_patch +patch_hash = ce568b4e1a7a593bb233f3ec97af88ceb850524cef802fe3f916b21d8a79b6b6 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/lz4_1.10.0-2/lz4-1.10.0.tgz +wrapdb_version = 1.10.0-2 + +[provide] +liblz4 = liblz4_dep diff --git a/src/numcodecs/_blosc_neon/shuffle-generic-neon.c b/subprojects/packagefiles/c-blosc/blosc/shuffle-generic-neon.c similarity index 100% rename from src/numcodecs/_blosc_neon/shuffle-generic-neon.c rename to subprojects/packagefiles/c-blosc/blosc/shuffle-generic-neon.c diff --git a/subprojects/packagefiles/c-blosc/meson.build b/subprojects/packagefiles/c-blosc/meson.build new file mode 100644 index 00000000..81aa299d --- /dev/null +++ b/subprojects/packagefiles/c-blosc/meson.build @@ -0,0 +1,88 @@ +# Meson build for c-blosc 1.x, maintained by numcodecs (upstream ships CMake only). +# Builds a static libblosc with lz4, zlib and zstd support. The codec libraries are +# resolved with dependency(), so they are shared with the parent project: either the +# same system libraries or the same WrapDB subprojects. +project('c-blosc', 'c', + default_options: ['c_std=gnu11', 'warning_level=1', 'default_library=static'], + meson_version: '>=1.6.0', +) + +cc = meson.get_compiler('c') + +lz4_dep = dependency('liblz4') +zlib_dep = dependency('zlib') +zstd_dep = dependency('libzstd') + +c_args = ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] +if host_machine.system() != 'windows' + c_args += ['-pthread'] +endif + +sources = files( + 'blosc/blosc.c', + 'blosc/blosclz.c', + 'blosc/fastcopy.c', + 'blosc/shuffle.c', + 'blosc/bitshuffle-generic.c', +) + +# --- SIMD shuffle --- +# x86: c-blosc selects SSE2/AVX2 at runtime via CPUID; these only control +# whether the kernels are compiled in. +have_sse2 = false +have_avx2 = false +have_neon = false + +if host_machine.cpu_family() == 'x86_64' + have_sse2 = not get_option('sse2').disabled() and cc.has_argument('-msse2') + have_avx2 = not get_option('avx2').disabled() and cc.has_argument('-mavx2') +endif + +# aarch64: c-blosc 1.x has no NEON shuffle, so shuffle-generic.c is replaced by a +# NEON implementation of the same symbols (kernels from c-blosc2). NEON is +# mandatory on aarch64, so no runtime detection is needed. +if host_machine.cpu_family() == 'aarch64' and not get_option('neon').disabled() + have_neon = cc.compiles(''' + #if !defined(__ARM_NEON) && !defined(_M_ARM64) + #error "no NEON" + #endif + #include + int main(void) { uint8x16_t v = vdupq_n_u8(0); return vgetq_lane_u8(v, 0); } + ''', name: 'NEON intrinsics') +endif + +if have_neon + sources += files('blosc/shuffle-generic-neon.c') +else + sources += files('blosc/shuffle-generic.c') +endif + +if have_sse2 + sources += files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c') + c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + c_args += ['-D__SSE2__'] + endif +endif + +if have_avx2 + sources += files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c') + c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + c_args += ['-D__AVX2__'] + endif +endif + +blosc_inc = include_directories('blosc') + +blosc_lib = static_library('blosc', sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: c_args, +) + +blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, +) +meson.override_dependency('blosc', blosc_dep) diff --git a/subprojects/packagefiles/c-blosc/meson.options b/subprojects/packagefiles/c-blosc/meson.options new file mode 100644 index 00000000..56da7eff --- /dev/null +++ b/subprojects/packagefiles/c-blosc/meson.options @@ -0,0 +1,6 @@ +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 SIMD shuffle') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 SIMD shuffle') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON SIMD shuffle on aarch64') diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 00000000..0626401a --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,14 @@ +[wrap-file] +directory = zlib-1.3.2 +source_url = https://zlib.net/zlib-1.3.2.tar.xz +source_fallback_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_source/zlib-1.3.2.tar.xz +source_filename = zlib-1.3.2.tar.xz +source_hash = d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 +patch_filename = zlib_1.3.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_patch +patch_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.2-1/zlib_1.3.2-1_patch.zip +patch_hash = 5ae7a2e92f823df118cfb8c1b23d94e3117864392b3446581d669049b2fba6dd +wrapdb_version = 1.3.2-1 + +[provide] +dependency_names = zlib diff --git a/subprojects/zstd.wrap b/subprojects/zstd.wrap new file mode 100644 index 00000000..fc6e4fcb --- /dev/null +++ b/subprojects/zstd.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = zstd-1.5.7 +source_url = https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz +source_filename = zstd-1.5.7.tar.gz +source_hash = eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 +patch_filename = zstd_1.5.7-3_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zstd_1.5.7-3/get_patch +patch_hash = 75ccb7b5bd26ae38b9d2db40ea4915504137835c15857ea833e70af1fe6f8686 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zstd_1.5.7-3/zstd-1.5.7.tar.gz +wrapdb_version = 1.5.7-3 + +[provide] +libzstd = libzstd_dep From e430dfa7bab0f560605ca50432b429cf2e099dcd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 13:48:45 +0200 Subject: [PATCH 3/4] build(c-blosc): use a trimmed, reproducible c-blosc source tarball The upstream c-blosc archive bundles zstd, lz4, zlib and snappy sources plus tests and compat fixtures, none of which the meson build uses (the codec libraries come from their own subprojects). Add tools/make_c_blosc_tarball.py, which extracts blosc/, the license files and the README into a deterministic tarball, and point c-blosc.wrap at that tarball. c-blosc now takes 0.07 MB of the sdist, which shrinks from 10.7 MB to 8.3 MB. Assisted-by: ClaudeCode:claude-opus-5-5 --- subprojects/c-blosc.wrap | 17 ++++--- tools/make_c_blosc_tarball.py | 92 +++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 tools/make_c_blosc_tarball.py diff --git a/subprojects/c-blosc.wrap b/subprojects/c-blosc.wrap index ba5d22d0..2dba0ba3 100644 --- a/subprojects/c-blosc.wrap +++ b/subprojects/c-blosc.wrap @@ -1,12 +1,17 @@ [wrap-file] # c-blosc 1.x is not in WrapDB, so this wrap is maintained here. It pins the # c-blosc commit numcodecs was previously pinned to as a git submodule (main -# after v1.21.6, needed for gcc 15). Switch to a release tarball once c-blosc -# tags a release that includes it. -directory = c-blosc-04c06fb7339d3df3600b1862077b8699b30a0e1d -source_url = https://github.com/Blosc/c-blosc/archive/04c06fb7339d3df3600b1862077b8699b30a0e1d.tar.gz -source_filename = c-blosc-04c06fb7339d3df3600b1862077b8699b30a0e1d.tar.gz -source_hash = 22c1d0fa6500f319db9a9da5e9344e3a2f26de98e3beadf25f96dd3c71a271f3 +# after v1.21.6, needed for gcc 15). +# +# The tarball is trimmed to what the meson build needs (no internal-complibs, +# tests or compat fixtures) to keep the sdist small. Regenerate it with +# python tools/make_c_blosc_tarball.py +# which prints the values below; the uncompressed tar's sha256 for this one is +# 6b7381ba006c825aea583b8aff9fcc8cdd9e7860ccc88bd29f415630742b28f4. +directory = c-blosc-04c06fb7339d-numcodecs +source_url = https://github.com/d-v-b/numcodecs/releases/download/vendor%2Fc-blosc-04c06fb7339d/c-blosc-04c06fb7339d-numcodecs.tar.gz +source_filename = c-blosc-04c06fb7339d-numcodecs.tar.gz +source_hash = ebadbf112007a44c46c4b0c303794e6369ccf8b0ddeb0600285cae0aca7f7a81 # meson.build/meson.options for c-blosc (upstream only ships CMake), plus # numcodecs-specific sources such as the NEON shuffle. patch_directory = c-blosc diff --git a/tools/make_c_blosc_tarball.py b/tools/make_c_blosc_tarball.py new file mode 100644 index 00000000..c7cb5dff --- /dev/null +++ b/tools/make_c_blosc_tarball.py @@ -0,0 +1,92 @@ +"""Build the trimmed c-blosc source tarball used by ``subprojects/c-blosc.wrap``. + +The upstream c-blosc archive bundles its own copies of zstd, lz4, zlib and snappy +(``internal-complibs/``), plus tests, benchmarks and compatibility fixtures. numcodecs +takes those codec libraries from their own subprojects, so shipping them again only +inflates the sdist. This script keeps just the files the meson build needs. + +The output is reproducible: members are sorted and all metadata (mtime, owner, +permissions) is normalized, so the uncompressed tar is byte-identical for a given +commit. The gzip layer is also deterministic for a given zlib, but may differ across +zlib implementations, so verify a published tarball by comparing ``tar sha256``. + +Usage:: + + python tools/make_c_blosc_tarball.py [-o OUTDIR] + +Then upload the tarball as a release asset and update ``source_url``, +``source_filename``, ``source_hash`` and ``directory`` in ``subprojects/c-blosc.wrap``. +""" + +import argparse +import gzip +import hashlib +import io +import tarfile +import urllib.request +from pathlib import Path + +ARCHIVE_URL = "https://github.com/Blosc/c-blosc/archive/{sha}.tar.gz" + +# Paths relative to the c-blosc repository root. Directories are included recursively. +KEEP = ("blosc/", "LICENSE.txt", "LICENSES/", "README.md") + + +def _keep(relpath: str) -> bool: + return any(relpath == k or (k.endswith("/") and relpath.startswith(k)) for k in KEEP) + + +def build(sha: str, outdir: Path) -> Path: + with urllib.request.urlopen(ARCHIVE_URL.format(sha=sha)) as resp: + archive = resp.read() + + src_prefix = f"c-blosc-{sha}/" + dst_prefix = f"c-blosc-{sha[:12]}-numcodecs/" + raw = io.BytesIO() + with ( + tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as upstream, + tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as out, + ): + files = sorted( + (m for m in upstream.getmembers() if m.isfile() and m.name.startswith(src_prefix)), + key=lambda m: m.name, + ) + files = [m for m in files if _keep(m.name[len(src_prefix) :])] + if not any(m.name.endswith("/blosc/blosc.c") for m in files): + raise RuntimeError("blosc/blosc.c not found; unexpected archive layout") + for m in files: + info = tarfile.TarInfo(dst_prefix + m.name[len(src_prefix) :]) + info.size = m.size + info.mode = 0o644 + info.mtime = 0 + info.uid = info.gid = 0 + info.uname = info.gname = "" + out.addfile(info, upstream.extractfile(m)) + tar_bytes = raw.getvalue() + + outdir.mkdir(parents=True, exist_ok=True) + path = outdir / f"{dst_prefix.rstrip('/')}.tar.gz" + gz = io.BytesIO() + with gzip.GzipFile(filename="", mode="wb", fileobj=gz, compresslevel=9, mtime=0) as f: + f.write(tar_bytes) + path.write_bytes(gz.getvalue()) + + print(f"wrote {path} ({path.stat().st_size} bytes, {len(files)} files)") + print(f"directory {dst_prefix.rstrip('/')}") + print(f"sha256 {hashlib.sha256(gz.getvalue()).hexdigest()} (source_hash)") + print(f"tar sha256 {hashlib.sha256(tar_bytes).hexdigest()} (uncompressed, for verification)") + return path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("sha", help="full c-blosc commit sha") + parser.add_argument("-o", "--outdir", type=Path, default=Path(".")) + args = parser.parse_args() + if len(args.sha) != 40: + parser.error("pass the full 40-character commit sha") + build(args.sha, args.outdir) + + +if __name__ == "__main__": + main() From 1879207368ab4db1a46aff78ee3df7275b9a2493 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 27 Sep 2026 13:55:02 +0200 Subject: [PATCH 4/4] ci: stop checking out the removed c-blosc submodule Drop the submodules settings from the GitHub Actions checkouts and the Read the Docs config now that c-blosc is a meson subproject, and add a release note for the migration. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci-i386.yml | 1 - .github/workflows/ci.yaml | 3 --- .github/workflows/wheel.yaml | 2 -- .readthedocs.yaml | 3 --- docs/release.md | 14 ++++++++++++++ 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 4ddba13e..bda2222a 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -18,7 +18,6 @@ jobs: - name: Checkout source uses: actions/checkout@v6 with: - submodules: recursive fetch-depth: 0 # required for version resolution - name: Setup Alpine Linux environment diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 645bf782..b56baaea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,6 @@ jobs: - name: Checkout source uses: actions/checkout@v6 with: - submodules: recursive fetch-depth: 0 # required for version resolution - name: Set up Python @@ -75,7 +74,6 @@ jobs: - name: Checkout source uses: actions/checkout@v6 with: - submodules: recursive fetch-depth: 0 - name: Set up Python @@ -125,7 +123,6 @@ jobs: - name: Checkout source uses: actions/checkout@v6 with: - submodules: recursive fetch-depth: 0 # required for version resolution - name: Set up Python diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index d8fa6ab1..6d59e0cd 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -32,7 +32,6 @@ jobs: steps: - uses: actions/checkout@v6 with: - submodules: true fetch-depth: 0 # required for version resolution for nightly wheels - uses: pypa/cibuildwheel@v3.4.1 @@ -48,7 +47,6 @@ jobs: steps: - uses: actions/checkout@v6 with: - submodules: true fetch-depth: 0 - uses: actions/setup-python@v6 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 44e97335..85be59cc 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,8 +1,5 @@ version: 2 -submodules: - include: all - build: os: ubuntu-24.04 tools: diff --git a/docs/release.md b/docs/release.md index fc1d3b59..90ba0223 100644 --- a/docs/release.md +++ b/docs/release.md @@ -24,6 +24,20 @@ option (default `auto`). By {user}`Davis Bennett `. +### Maintenance + +* **Replace the c-blosc git submodule with meson wrap subprojects.** zstd, lz4 and zlib + now come from [WrapDB](https://mesonbuild.com/Wrapdb-projects.html) and are pinned + independently of c-blosc (zstd 1.5.6 -> 1.5.7, zlib 1.3.1 -> 1.3.2). c-blosc is pinned + in `subprojects/c-blosc.wrap` to a trimmed source tarball generated by + `tools/make_c_blosc_tarball.py`, with its meson build and numcodecs patches in + `subprojects/packagefiles/c-blosc/`. Cloning no longer needs `--recursive`; the first + build downloads the sources, and the sdist includes them so it still builds offline. + Building from source now also requires a C++ compiler (zstd's meson build declares + C++). The `system_*` meson options are unchanged. License texts for the bundled + libraries, including the previously missing zstd license, are in `LICENSES/`. + By {user}`Davis Bennett `. + (release_0.17.0)= * Allow building against a system Zlib. Note, this is only required if blosc is still