Add free-threaded CPython (3.14t) support

.github/workflows/ci.yaml

@@ -12,13 +12,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12", "3.13", "3.14"]
+        python-version: ["3.12", "3.13", "3.14", "3.14t"]
         # macos-15-large is an intel runner, macos-14 is an arm64 runner
         platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14]
 
     defaults:
       run:
-        shell: bash -el {0}
+        shell: bash -e {0}
 
     steps:
       - name: Checkout source
@@ -34,13 +34,18 @@ jobs:
           cache: 'pip'
 
       - name: Install numcodecs
+        if: ${{ !endsWith(matrix.python-version, 't') }}
         run: python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]"
 
+      - name: Install numcodecs (free-threaded)
+        # pcodec, zfpy and google_crc32c do not ship free-threaded wheels yet
+        if: ${{ endsWith(matrix.python-version, 't') }}
+        run: python -m pip install -v ".[test,test_extras,msgpack,crc32c]"
+
       - name: List installed packages
         run: python -m pip list
 
       - name: Run tests
-        shell: "bash -l {0}"
         run: pytest -v
 
       - uses: codecov/codecov-action@v6

.github/workflows/wheel.yaml

@@ -22,8 +22,10 @@ jobs:
         # macos-15-large is an intel runner, macos-14 is an arm64 runner
         os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-15-large, macos-14]
     env:
-      CIBW_TEST_COMMAND: python -c "import numcodecs"
-      CIBW_BUILD: "cp312-* cp313-* cp314-*"
+      # on free-threaded builds also check that no extension re-enables the GIL
+      CIBW_TEST_COMMAND: python -c "import sys, sysconfig, numcodecs; assert not (sysconfig.get_config_var('Py_GIL_DISABLED') and sys._is_gil_enabled())"
+      CIBW_BUILD: "cp312-* cp313-* cp314-* cp314t-*"
+      CIBW_ENABLE: cpython-freethreading
       CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x"
       # note: cibuildwheel config-settings are set in pyproject.toml
 

docs/release.md

@@ -14,6 +14,22 @@
 
 ## Unreleased
 
+### Enhancements
+
+* Add support for free-threaded CPython (3.14t). All Cython extensions are declared
+  `freethreading_compatible`, so importing `numcodecs` no longer re-enables the GIL;
+  free-threaded wheels are now built and 3.14t is part of the test matrix. Building from
+  source now requires Cython 3.1 or newer.
+
+  Two behavioral changes in `numcodecs.blosc` accompany this: the undocumented
+  `numcodecs.blosc.get_mutex` function is removed, replaced by a private
+  `threading.Lock` that also guards `set_nthreads`, `_init`, `_destroy` and the
+  global-context decompression path; and on platforms where `multiprocessing.Lock()`
+  raises (no `sem_open`), Blosc's internal threads are now used from the main thread
+  instead of always falling back to the single-threaded context functions.
+
+  By {user}`Kumar Aditya <kumaraditya303>`, {issue}`858`
+
 ### Maintenance
 
 * **Migrate build system from setuptools/setup.py to meson-python.** This replaces the

pyproject.toml

@@ -3,7 +3,7 @@ requires = [
     "meson-python>=0.17",
     "meson>=1.6.0",
     "setuptools-scm>=6.2",
-    "Cython>=3.0",
+    "Cython>=3.1",
     "numpy>=2",
 ]
 build-backend = "mesonpy"
@@ -84,7 +84,7 @@ dev = [
     "meson-python>=0.17",
     "meson>=1.6.0",
     "ninja",
-    "cython>=3.0",
+    "cython>=3.1",
     "setuptools-scm>=6.2",
     "numpy>=2",
     "pytest==9.0.3",

src/numcodecs/_shuffle.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 
 cimport cython
 

src/numcodecs/blosc.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 import threading
 import multiprocessing
 import os
@@ -72,54 +73,55 @@ AUTOSHUFFLE = -1
 # automatic block size - let blosc decide
 AUTOBLOCKS = 0
 
-# synchronization
-_MUTEX = None
-_MUTEX_IS_INIT = False
-
-def get_mutex():
-    global _MUTEX_IS_INIT, _MUTEX
-    if not _MUTEX_IS_INIT:
-        try:
-            mutex = multiprocessing.Lock()
-        except OSError:
-            mutex = None
-        except ImportError:
-            mutex = None
-        _MUTEX = mutex
-        _MUTEX_IS_INIT = True
-    return _MUTEX
+# computed once at import; blosc_list_compressors() fills a static buffer without locking
+_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(','))
+
+# serializes access to blosc's global context
+_MUTEX = threading.Lock()
+
+
+def _reset_mutex_after_fork():
+    global _MUTEX
+    _MUTEX = threading.Lock()
+
+
+if hasattr(os, 'register_at_fork'):
+    os.register_at_fork(after_in_child=_reset_mutex_after_fork)
 
 # store ID of process that first loads the module, so we can detect a fork later
 _importer_pid = os.getpid()
 
 
 def _init():
     """Initialize the Blosc library environment."""
-    blosc_init()
+    with _MUTEX:
+        blosc_init()
 
 
 def _destroy():
     """Destroy the Blosc library environment."""
-    blosc_destroy()
+    with _MUTEX:
+        blosc_destroy()
 
 
 def list_compressors():
     """Get a list of compressors supported in the current build."""
-    s = blosc_list_compressors()
-    s = s.decode('ascii')
-    return s.split(',')
+    return list(_COMPRESSORS)
 
 
 def get_nthreads():
     """Get the number of threads that Blosc uses internally for compression and
     decompression."""
-    return blosc_get_nthreads()
+    with _MUTEX:
+        return blosc_get_nthreads()
 
 
 def set_nthreads(int nthreads):
     """Set the number of threads that Blosc uses internally for compression and
     decompression."""
-    return blosc_set_nthreads(nthreads)
+    # blosc_set_nthreads re-creates the global context
+    with _MUTEX:
+        return blosc_set_nthreads(nthreads)
 
 
 def _cbuffer_sizes(source):
@@ -247,7 +249,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
 
     # check valid cname early
     cname_str = cname.decode('ascii')
-    if cname_str not in list_compressors():
+    if cname_str not in _COMPRESSORS:
         _err_bad_cname(cname_str)
 
     # obtain source memoryview
@@ -288,8 +290,8 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
 
             # N.B., we are using blosc's global context, and so we need to use a lock
             # to ensure no-one else can modify the global context while we're setting it
-            # up and using it.
-            with get_mutex():
+            # up and using it (including set_nthreads/_destroy re-initializing it).
+            with _MUTEX:
 
                 # set compressor
                 compressor_set = blosc_set_compressor(cname)
@@ -385,8 +387,9 @@ def decompress(source, dest=None):
         # perform decompression
         if _get_use_threads():
             # allow blosc to use threads internally
-            with nogil:
-                ret = blosc_decompress(source_ptr, dest_ptr, nbytes)
+            with _MUTEX:
+                with nogil:
+                    ret = blosc_decompress(source_ptr, dest_ptr, nbytes)
         else:
             with nogil:
                 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
@@ -410,10 +413,6 @@ def _get_use_threads():
     global use_threads
     proc = multiprocessing.current_process()
 
-    # check if locks are available, and if not no threads
-    if not get_mutex():
-        return False
-
     # check for fork
     if proc.pid != _importer_pid:
         # If this module has been imported in the parent process, and the current process

src/numcodecs/compat_ext.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 
 from cpython.buffer cimport PyBuffer_IsContiguous
 from cpython.memoryview cimport PyMemoryView_GET_BUFFER

src/numcodecs/fletcher32.pyx

@@ -1,6 +1,7 @@
 # cython: language_level=3
 # cython: overflowcheck=False
 # cython: cdivision=True
+# cython: freethreading_compatible=True
 
 
 from libc.stdint cimport uint8_t, uint16_t, uint32_t

src/numcodecs/jenkins.pyx

@@ -1,6 +1,7 @@
 # cython: language_level=3
 # cython: overflowcheck=False
 # cython: cdivision=True
+# cython: freethreading_compatible=True
 
 """
 Cython implementation of Bob Jenkin's hashlittle from lookup3.c.

src/numcodecs/lz4.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 
 
 from libc.stdint cimport uint8_t, uint32_t

src/numcodecs/vlen.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 
 
 cimport cython

src/numcodecs/zstd.pyx

@@ -3,6 +3,7 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
+# cython: freethreading_compatible=True
 
 
 from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize

tests/test_blosc.py

@@ -1,3 +1,5 @@
+import threading
+import time
 from multiprocessing import Pool
 from multiprocessing.pool import ThreadPool
 
@@ -238,6 +240,45 @@ def test_multiprocessing(use_threads, pool):
         blosc.use_threads = None  # restore default
 
 
+def test_set_nthreads_races_global_context():
+    # Regression test: blosc_set_nthreads re-creates Blosc's global context, so it must
+    # be serialized against compress/decompress calls that use that context.
+    data = np.arange(200_000, dtype='i4')
+    stop = threading.Event()
+    errors = []
+
+    def worker():
+        try:
+            while not stop.is_set():
+                enc = blosc.compress(data, b'lz4', 5)
+                assert blosc.decompress(enc) == data.tobytes()
+        except Exception as e:
+            errors.append(e)
+
+    def toggler():
+        n = 1
+        while not stop.is_set():
+            n = 1 + (n % 4)
+            blosc.set_nthreads(n)
+
+    # force every thread onto the global-context path
+    blosc.use_threads = True
+    original_nthreads = blosc.get_nthreads()
+    try:
+        threads = [threading.Thread(target=worker) for _ in range(8)]
+        threads.append(threading.Thread(target=toggler))
+        for t in threads:
+            t.start()
+        time.sleep(1)
+        stop.set()
+        for t in threads:
+            t.join()
+    finally:
+        blosc.use_threads = None  # restore default
+        blosc.set_nthreads(original_nthreads)
+    assert not errors
+
+
 def test_err_decode_object_buffer():
     check_err_decode_object_buffer(Blosc())
 

tests/test_free_threading.py

@@ -0,0 +1,25 @@
+import subprocess
+import sys
+import sysconfig
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+    not sysconfig.get_config_var("Py_GIL_DISABLED"),
+    reason="requires a free-threaded Python build",
+)
+
+
+def test_import_does_not_reenable_gil():
+    # Run in a fresh interpreter so that no other extension module (pytest plugins,
+    # optional dependencies) can have re-enabled the GIL already.
+    code = "import sys, numcodecs; sys.exit(int(sys._is_gil_enabled()))"
+    proc = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert proc.returncode == 0, (
+        "importing numcodecs re-enabled the GIL on a free-threaded build\n" + proc.stderr
+    )