diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53f41dc85..b5bb507cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,13 +12,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14", "3.14t"] # macos-15-large is an intel runner, macos-14 is an arm64 runner platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] defaults: run: - shell: bash -el {0} + shell: bash -e {0} steps: - name: Checkout source @@ -34,13 +34,18 @@ jobs: cache: 'pip' - name: Install numcodecs + if: ${{ !endsWith(matrix.python-version, 't') }} run: python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + - name: Install numcodecs (free-threaded) + # pcodec, zfpy and google_crc32c do not ship free-threaded wheels yet + if: ${{ endsWith(matrix.python-version, 't') }} + run: python -m pip install -v ".[test,test_extras,msgpack,crc32c]" + - name: List installed packages run: python -m pip list - name: Run tests - shell: "bash -l {0}" run: pytest -v - uses: codecov/codecov-action@v6 diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 947650e40..d8fa6ab11 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -22,8 +22,10 @@ jobs: # macos-15-large is an intel runner, macos-14 is an arm64 runner os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14] env: - CIBW_TEST_COMMAND: python -c "import numcodecs" - CIBW_BUILD: "cp312-* cp313-* cp314-*" + # on free-threaded builds also check that no extension re-enables the GIL + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, numcodecs; assert not (sysconfig.get_config_var('Py_GIL_DISABLED') and sys._is_gil_enabled())" + CIBW_BUILD: "cp312-* cp313-* cp314-* cp314t-*" + CIBW_ENABLE: cpython-freethreading CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x" # note: cibuildwheel config-settings are set in pyproject.toml diff --git a/docs/release.md b/docs/release.md index 78e8901c5..2a4c3b4dc 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,22 @@ ## Unreleased +### Enhancements + +* Add support for free-threaded CPython (3.14t). All Cython extensions are declared + `freethreading_compatible`, so importing `numcodecs` no longer re-enables the GIL; + free-threaded wheels are now built and 3.14t is part of the test matrix. Building from + source now requires Cython 3.1 or newer. + + Two behavioral changes in `numcodecs.blosc` accompany this: the undocumented + `numcodecs.blosc.get_mutex` function is removed, replaced by a private + `threading.Lock` that also guards `set_nthreads`, `_init`, `_destroy` and the + global-context decompression path; and on platforms where `multiprocessing.Lock()` + raises (no `sem_open`), Blosc's internal threads are now used from the main thread + instead of always falling back to the single-threaded context functions. + + By {user}`Kumar Aditya `, {issue}`858` + ### Maintenance * **Migrate build system from setuptools/setup.py to meson-python.** This replaces the diff --git a/pyproject.toml b/pyproject.toml index ce472c983..185ecc3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "meson-python>=0.17", "meson>=1.6.0", "setuptools-scm>=6.2", - "Cython>=3.0", + "Cython>=3.1", "numpy>=2", ] build-backend = "mesonpy" @@ -84,7 +84,7 @@ dev = [ "meson-python>=0.17", "meson>=1.6.0", "ninja", - "cython>=3.0", + "cython>=3.1", "setuptools-scm>=6.2", "numpy>=2", "pytest==9.0.3", diff --git a/src/numcodecs/_shuffle.pyx b/src/numcodecs/_shuffle.pyx index 0f0dafeb8..667beb5ab 100644 --- a/src/numcodecs/_shuffle.pyx +++ b/src/numcodecs/_shuffle.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f73588..8c52dd95c 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True import threading import multiprocessing import os @@ -72,22 +73,20 @@ AUTOSHUFFLE = -1 # automatic block size - let blosc decide AUTOBLOCKS = 0 -# synchronization -_MUTEX = None -_MUTEX_IS_INIT = False - -def get_mutex(): - global _MUTEX_IS_INIT, _MUTEX - if not _MUTEX_IS_INIT: - try: - mutex = multiprocessing.Lock() - except OSError: - mutex = None - except ImportError: - mutex = None - _MUTEX = mutex - _MUTEX_IS_INIT = True - return _MUTEX +# computed once at import; blosc_list_compressors() fills a static buffer without locking +_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(',')) + +# serializes access to blosc's global context +_MUTEX = threading.Lock() + + +def _reset_mutex_after_fork(): + global _MUTEX + _MUTEX = threading.Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_mutex_after_fork) # store ID of process that first loads the module, so we can detect a fork later _importer_pid = os.getpid() @@ -95,31 +94,34 @@ _importer_pid = os.getpid() def _init(): """Initialize the Blosc library environment.""" - blosc_init() + with _MUTEX: + blosc_init() def _destroy(): """Destroy the Blosc library environment.""" - blosc_destroy() + with _MUTEX: + blosc_destroy() def list_compressors(): """Get a list of compressors supported in the current build.""" - s = blosc_list_compressors() - s = s.decode('ascii') - return s.split(',') + return list(_COMPRESSORS) def get_nthreads(): """Get the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_get_nthreads() + with _MUTEX: + return blosc_get_nthreads() def set_nthreads(int nthreads): """Set the number of threads that Blosc uses internally for compression and decompression.""" - return blosc_set_nthreads(nthreads) + # blosc_set_nthreads re-creates the global context + with _MUTEX: + return blosc_set_nthreads(nthreads) def _cbuffer_sizes(source): @@ -247,7 +249,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # check valid cname early cname_str = cname.decode('ascii') - if cname_str not in list_compressors(): + if cname_str not in _COMPRESSORS: _err_bad_cname(cname_str) # obtain source memoryview @@ -288,8 +290,8 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, # N.B., we are using blosc's global context, and so we need to use a lock # to ensure no-one else can modify the global context while we're setting it - # up and using it. - with get_mutex(): + # up and using it (including set_nthreads/_destroy re-initializing it). + with _MUTEX: # set compressor compressor_set = blosc_set_compressor(cname) @@ -385,8 +387,9 @@ def decompress(source, dest=None): # perform decompression if _get_use_threads(): # allow blosc to use threads internally - with nogil: - ret = blosc_decompress(source_ptr, dest_ptr, nbytes) + with _MUTEX: + with nogil: + ret = blosc_decompress(source_ptr, dest_ptr, nbytes) else: with nogil: ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) @@ -410,10 +413,6 @@ def _get_use_threads(): global use_threads proc = multiprocessing.current_process() - # check if locks are available, and if not no threads - if not get_mutex(): - return False - # check for fork if proc.pid != _importer_pid: # If this module has been imported in the parent process, and the current process diff --git a/src/numcodecs/compat_ext.pyx b/src/numcodecs/compat_ext.pyx index 05dc2bf42..9671afde0 100644 --- a/src/numcodecs/compat_ext.pyx +++ b/src/numcodecs/compat_ext.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.buffer cimport PyBuffer_IsContiguous from cpython.memoryview cimport PyMemoryView_GET_BUFFER diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 56a5f793c..1ecbb45d6 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint16_t, uint32_t diff --git a/src/numcodecs/jenkins.pyx b/src/numcodecs/jenkins.pyx index 0efaf26ce..acdaa1c1b 100644 --- a/src/numcodecs/jenkins.pyx +++ b/src/numcodecs/jenkins.pyx @@ -1,6 +1,7 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True +# cython: freethreading_compatible=True """ Cython implementation of Bob Jenkin's hashlittle from lookup3.c. diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 9b1b431c8..6eac246b5 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from libc.stdint cimport uint8_t, uint32_t diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index 69ae620a8..f51e8cad7 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True cimport cython diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7e070a62a..7b6ff3c02 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -3,6 +3,7 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 +# cython: freethreading_compatible=True from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 537dc3505..b920f03e8 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -1,3 +1,5 @@ +import threading +import time from multiprocessing import Pool from multiprocessing.pool import ThreadPool @@ -238,6 +240,45 @@ def test_multiprocessing(use_threads, pool): blosc.use_threads = None # restore default +def test_set_nthreads_races_global_context(): + # Regression test: blosc_set_nthreads re-creates Blosc's global context, so it must + # be serialized against compress/decompress calls that use that context. + data = np.arange(200_000, dtype='i4') + stop = threading.Event() + errors = [] + + def worker(): + try: + while not stop.is_set(): + enc = blosc.compress(data, b'lz4', 5) + assert blosc.decompress(enc) == data.tobytes() + except Exception as e: + errors.append(e) + + def toggler(): + n = 1 + while not stop.is_set(): + n = 1 + (n % 4) + blosc.set_nthreads(n) + + # force every thread onto the global-context path + blosc.use_threads = True + original_nthreads = blosc.get_nthreads() + try: + threads = [threading.Thread(target=worker) for _ in range(8)] + threads.append(threading.Thread(target=toggler)) + for t in threads: + t.start() + time.sleep(1) + stop.set() + for t in threads: + t.join() + finally: + blosc.use_threads = None # restore default + blosc.set_nthreads(original_nthreads) + assert not errors + + def test_err_decode_object_buffer(): check_err_decode_object_buffer(Blosc()) diff --git a/tests/test_free_threading.py b/tests/test_free_threading.py new file mode 100644 index 000000000..cf0d1e177 --- /dev/null +++ b/tests/test_free_threading.py @@ -0,0 +1,25 @@ +import subprocess +import sys +import sysconfig + +import pytest + +pytestmark = pytest.mark.skipif( + not sysconfig.get_config_var("Py_GIL_DISABLED"), + reason="requires a free-threaded Python build", +) + + +def test_import_does_not_reenable_gil(): + # Run in a fresh interpreter so that no other extension module (pytest plugins, + # optional dependencies) can have re-enabled the GIL already. + code = "import sys, numcodecs; sys.exit(int(sys._is_gil_enabled()))" + proc = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, ( + "importing numcodecs re-enabled the GIL on a free-threaded build\n" + proc.stderr + )