sgkit zarr 3 migration

.github/workflows/build-gpu.yml

@@ -49,7 +49,7 @@ jobs:
         run: |
           conda install -c nvidia cudatoolkit
           # Avoid building SciPy from source in CI.
-          uv pip install --only-binary scipy --python "$CONDA_PREFIX/bin/python" -e ".[bgen,plink]" --group test
+          uv pip install --only-binary scipy --python "$CONDA_PREFIX/bin/python" -e ".[plink]" --group test
 
       - name: Numba Information
         run: |

.github/workflows/build.yml

@@ -41,14 +41,11 @@ jobs:
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-zarr-version:
-    name: Test Zarr Python v3
+  test-xarray-minimum:
+    name: Test minimum Xarray
     # Scheduled runs only on the origin org
     if: (github.event_name == 'schedule' && github.repository_owner == 'sgkit-dev') || (github.event_name != 'schedule')
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        zarr: [">=3"]
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-python@v7
@@ -63,10 +60,9 @@ jobs:
       - name: Install dependencies
         run: |
           uv sync --locked --all-extras --group test --no-default-groups --python 3.11
-      - name: Install zarr${{ matrix.zarr }}
+      - name: Install minimum supported Xarray
         run: |
-          uv pip install --python .venv --pre 'zarr${{ matrix.zarr }}'
-          uv pip uninstall --python .venv bio2zarr  # TODO: remove when bio2zarr supports Zarr Python 3
+          uv pip install --python .venv 'xarray==2026.9.0'
       - name: Run tests
         run: |
           uv run --no-sync pytest

.github/workflows/wheels.yml

@@ -81,9 +81,8 @@ jobs:
           # Avoid building SciPy from source in CI.
           wheel=$(ls artifact/sgkit-*.whl)
           uv venv --python "${{ matrix.python-version }}"
-          uv pip install --python .venv --only-binary scipy "${wheel}[bgen,plink]"
+          uv pip install --python .venv --only-binary scipy "${wheel}[plink]"
           uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit.py
-          uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit_bgen.py
           uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit_plink.py
 
   windows-test:
@@ -120,9 +119,8 @@ jobs:
           # Avoid building SciPy from source in CI.
           $wheel = (Get-ChildItem artifact/sgkit-*.whl).FullName
           uv venv --python "${{ matrix.python-version }}"
-          uv pip install --python .venv --only-binary scipy "${wheel}[bgen,plink]"
+          uv pip install --python .venv --only-binary scipy "${wheel}[plink]"
           uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit.py
-          uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit_bgen.py
           uv run --no-sync python sgkit-copy/.github/scripts/test_sgkit_plink.py
 
 

README.md

@@ -10,6 +10,14 @@ general-purpose frameworks such as [Xarray](http://xarray.pydata.org/en/stable/)
 
 For more information on using sgkit, see the [documentation](https://sgkit-dev.github.io/sgkit/).
 
+**Genetic file conversion:** Use [bio2zarr](https://sgkit-dev.github.io/bio2zarr/)
+to convert genetic file formats to Zarr, then load them with `sgkit.load_dataset`.
+The BGEN reader, conversion functions and `bgen` extra have been removed;
+sgkit no longer depends on cbgen. Use a bio2zarr version with BGEN support
+(tracked in [bio2zarr issue 417](https://github.com/sgkit-dev/bio2zarr/issues/417)).
+See the [BGEN migration guide](docs/user_guide.rst#bgen) for details and
+`unpack_variables`, retained only for existing legacy packed Zarr stores.
+
 [//]: # (numfocus-fiscal-sponsor-attribution)
 
 The sgkit project uses a [custom governance model](./GOVERNANCE.md) 

docs/api.rst

@@ -9,16 +9,24 @@ IO/imports and exports
 
 See :ref:`reading_and_writing_genetic_data`
 
+.. important::
+
+   **Use bio2zarr for genetic file format conversion to Zarr**, then load the
+   converted dataset with :func:`sgkit.load_dataset`. The BGEN reader and
+   conversion functions have been removed. See :ref:`bgen_conversion_migration`
+   for migration and legacy-store unpacking guidance.
+
 BGEN
 -----
 
+Only the helper for existing legacy packed stores remains; it does not read
+BGEN files or require an optional backend.
+
 .. currentmodule:: sgkit.io.bgen
 .. autosummary::
    :toctree: generated/
 
-   bgen_to_zarr
-   read_bgen
-   rechunk_bgen
+   unpack_variables
 
 PLINK
 -----

docs/changelog.rst

@@ -11,26 +11,49 @@ Changelog
 .. New Features
 .. ~~~~~~~~~~~~
 
-.. Breaking changes
-.. ~~~~~~~~~~~~~~~~
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+- Require Zarr Python >= 3 and Xarray >= 2026.9.0. The default on-disk Zarr
+  format remains 2. (:issue:`1355`)
+- **Remove BGEN reading and conversion**: ``sgkit.io.bgen.read_bgen``,
+  ``sgkit.io.bgen.bgen_to_zarr`` and ``sgkit.io.bgen.rechunk_bgen``, along with
+  the ``bgen`` extra and the cbgen and rechunker dependencies.
+  **Use bio2zarr for genetic file format conversion to Zarr**, and load the
+  result with :func:`load_dataset`. BGEN input requires a bio2zarr version
+  with BGEN support
+  (`bio2zarr issue 417 <https://github.com/sgkit-dev/bio2zarr/issues/417>`_).
+  The dependency-free :func:`sgkit.io.bgen.unpack_variables` helper remains
+  for existing legacy packed stores. See :ref:`bgen_conversion_migration`.
+  (:pr:`1325`, :issue:`1355`)
 
 Deprecations
 ~~~~~~~~~~~~
 
+- BGEN reading and conversion have moved out of sgkit. Use
+  `bio2zarr <https://github.com/sgkit-dev/bio2zarr>`_ for BGEN-to-Zarr conversion
+  and :func:`load_dataset` for analysis. Unlike the deprecated PLINK functions
+  below, ``read_bgen``, ``bgen_to_zarr`` and ``rechunk_bgen`` are already removed
+  in this release; see Breaking changes above and
+  :ref:`bgen_conversion_migration`. Only the dependency-free
+  :func:`sgkit.io.bgen.unpack_variables` helper remains for legacy packed stores.
+  (:pr:`1325`, :issue:`1331`, :issue:`1355`)
 - Functions for reading and writing plink are deprecated, please use the `bio2zarr <https://github.com/sgkit-dev/bio2zarr>`_
   and `vcztools <https://github.com/sgkit-dev/vcztools>`_ packages.
   (:user:`tomwhite`, :pr:`1324`, :issue:`1318`)
-- Using rechunker for rechunking in sgkit is deprecated, please use the the
-  `Xarray chunk <https://docs.xarray.dev/en/stable/generated/xarray.Dataset.chunk.html#xarray.Dataset.chunk>`_
-  function directly.
-  (:user:`tomwhite`, :pr:`1325`)
 
 .. Improvements
 .. ~~~~~~~~~~~~
 
 Bug fixes
 ~~~~~~~~~
 
+- Unpacking legacy BGEN-derived Zarr datasets no longer imports or requires
+  cbgen. Use :func:`sgkit.io.bgen.unpack_variables`; the original
+  ``sgkit.io.bgen.bgen_reader.unpack_variables`` import is also preserved.
+  Its documentation explains the legacy packing convention and when unpacking
+  is needed; ordinary unpacked datasets and new bio2zarr output do not need it.
+  (:issue:`1331`)
 - Fixed bug causing high memory usage in :func:`pedigree_kinship` when
   evaluating a chunked relationship matrix with the Hamilton-Kerr method.
   (:user:`timothymillar`, :pr:`1335`, :issue:`1333`)

docs/conf.py

@@ -117,7 +117,11 @@ def filter(self, record: pylogging.LogRecord) -> bool:
 
 autosummary_generate = True
 
-nitpick_ignore = [("py:class", "sgkit.display.GenotypeDisplay")]
+nitpick_ignore = [
+    ("py:class", "sgkit.display.GenotypeDisplay"),
+    # Zarr's StoreLike alias is rendered as an undocumented TypeAliasType.
+    ("py:class", "zarr.storage._common.TypeAliasType"),
+]
 
 
 # FIXME: Workaround for linking xarray module
@@ -130,6 +134,7 @@ def filter(self, record: pylogging.LogRecord) -> bool:
 
 intersphinx_mapping = dict(
     dask=("https://docs.dask.org/en/stable/", None),
+    fsspec=("https://filesystem-spec.readthedocs.io/en/latest/", None),
     xarray=("https://docs.xarray.dev/en/stable/", None),
     zarr=("https://zarr.readthedocs.io/en/stable", None),
     numpy=("https://numpy.org/doc/stable/", None),

docs/user_guide.rst

@@ -13,43 +13,45 @@ User Guide
 Reading and writing genetic data
 ================================
 
+.. important::
+
+   **Use bio2zarr for genetic file format conversion to Zarr**, then use
+   :func:`sgkit.load_dataset` for analysis in sgkit. See the
+   `bio2zarr documentation <https://sgkit-dev.github.io/bio2zarr/>`_ for
+   installation and conversion instructions.
+
+   BGEN reading and conversion have been removed from sgkit, along with the
+   optional cbgen dependency. See :ref:`bgen_conversion_migration` for conversion
+   guidance and support for existing legacy packed stores.
+
 .. _installation:
 
 Installation
 ------------
 
-Sgkit can read standard genetic file formats, including PLINK and BGEN. For reading VCF,
-please use the `bio2zarr <https://github.com/sgkit-dev/bio2zarr>`_ package.
+Sgkit reads Zarr datasets directly. For VCF, BGEN and PLINK conversion, use
+`bio2zarr <https://github.com/sgkit-dev/bio2zarr>`_. The deprecated direct PLINK
+reader remains available with optional dependencies.
 
-If sgkit has been installed using conda, support for reading BGEN and PLINK is included.
+If sgkit has been installed using conda, support for reading PLINK is included.
 
-If sgkit has been installed using pip, then support for reading these formats is
+If sgkit has been installed using pip, then support for reading PLINK is
 not included, and requires additional dependencies, which can be installed
 as an "extra" feature using pip, as follows.
 
-To install sgkit with BGEN support::
-
-    $ pip install 'sgkit[bgen]'
-
 To install sgkit with PLINK support::
 
     $ pip install 'sgkit[plink]'
 
 Converting genetic data to Zarr
 -------------------------------
 
-There are broadly two ways of loading genetic file format ``X`` for use in sgkit:
-
-1. a ``read_X`` function to read the file directly
-2. a ``X_to_zarr`` function to convert to Zarr first
-
-Which one to use depends on the size of the input, as well as the file format (not all file
-formats have both options).
-
-Generally speaking, the ``read_X`` functions are convenience functions that are suitable
-for small input files, or where the cost of conversion is small. The ``X_to_zarr`` functions
-are more appropriate for large datasets, since the expensive conversion step need only be
-paid once.
+Genetic file format conversion belongs in
+`bio2zarr <https://sgkit-dev.github.io/bio2zarr/>`__, rather than sgkit.
+Convert inputs with bio2zarr before analysis, so the expensive conversion step
+need only be paid once. BGEN input requires a bio2zarr version with BGEN support
+(see :ref:`bgen_conversion_migration`). The remaining direct PLINK functions
+are deprecated convenience functions for small inputs.
 
 After converting a file to Zarr format, it can be loaded with sgkit's :func:`sgkit.load_dataset`.
 This is like Xarray's :func:`xarray.open_zarr` function, with some useful defaults applied for you.
@@ -58,22 +60,76 @@ Similarly, the function :func:`sgkit.save_dataset` can be used to save the datas
 This can be used to save a newly-loaded dataset, or a dataset with new variables that are the
 result of running genetic methods on the data, so it can be loaded again later.
 
-By default, Zarr datasets created by sgkit (and Xarray) have `consolidated metadata <http://xarray.pydata.org/en/stable/user-guide/io.html#consolidated-metadata>`_,
+Sgkit requires Zarr Python >= 3 and Xarray >= 2026.9.0, but
+:func:`sgkit.save_dataset` continues to write **Zarr format 2** by default.
+The Python package version does not determine the on-disk format.
+Pass ``zarr_format=3`` to opt into format 3; :func:`sgkit.load_dataset`
+can read either format.
+The Xarray minimum includes support for the native NumPy string dtype returned
+by recent Zarr releases, preserving string-array chunking on load.
+
+By default, Zarr format 2 datasets created by sgkit have `consolidated metadata <http://xarray.pydata.org/en/stable/user-guide/io.html#consolidated-metadata>`_,
 which makes opening the dataset faster.
 
+.. _bgen_conversion_migration:
+
 BGEN
 ----
 
-The :func:`sgkit.io.bgen.read_bgen` function loads a single BGEN dataset as Dask
-arrays within an :class:`xarray.Dataset` from a ``bgen`` file.
+.. warning::
+
+   ``sgkit.io.bgen.read_bgen``, ``sgkit.io.bgen.bgen_to_zarr`` and
+   ``sgkit.io.bgen.rechunk_bgen`` have been removed, together with the ``bgen``
+   extra. Sgkit no longer reads BGEN files or depends on cbgen.
+
+Use bio2zarr for BGEN conversion, then load the resulting Zarr dataset with
+:func:`sgkit.load_dataset`. Install a bio2zarr version that includes BGEN support;
+availability is tracked in
+`bio2zarr issue 417 <https://github.com/sgkit-dev/bio2zarr/issues/417>`_.
+An older bio2zarr release without BGEN support cannot replace the removed reader.
+Sgkit does not provide an alternative BGEN conversion path.
 
-The :func:`sgkit.io.bgen.bgen_to_zarr` function converts a ``bgen`` file to Zarr
-files stored in sgkit's Xarray data representation, which can then be opened
-as a :class:`xarray.Dataset`.
+Legacy packed stores
+~~~~~~~~~~~~~~~~~~~~
+
+Some older sgkit BGEN converters packed variables to reduce storage size:
+they saved only the heterozygous and homozygous-alternate probabilities,
+omitted dosage, and used one missingness flag per variant/sample pair.
+For these **known legacy packed stores only**, use
+:func:`sgkit.io.bgen.unpack_variables` after loading and before methods that
+expect complete genotype probabilities or dosage. This helper operates on an
+existing Xarray dataset and needs no BGEN backend::
+
+    import sgkit as sg
+    from sgkit.io.bgen import unpack_variables
+
+    packed = sg.load_dataset("legacy-packed.zarr")
+    ds = unpack_variables(packed)
+
+The expected signature is ``call_genotype_probability`` with dimensions
+``(variants, samples, genotypes)`` and **two** stored probabilities, plus
+``call_genotype_probability_mask`` with dimensions ``(variants, samples)``.
+Unpacking restores three diploid, biallelic genotype probabilities, computes
+alternate-allele dosage, and expands the probability mask. For example,
+``[0.25, 0.5]`` becomes ``[0.25, 0.25, 0.5]`` with dosage ``1.25``.
+Missing probabilities remain NaN. The input is unchanged and Dask arrays
+remain lazy; lost quantization precision cannot be recovered.
+
+**Do not unpack ordinary datasets, new bio2zarr output, or an already-unpacked
+store.** A two-state probability array alone is not evidence of legacy packing:
+other ploidies or allele counts can also produce two states. The helper is not
+a general BGEN decoder or schema adapter. :func:`sgkit.load_dataset` does not
+automatically detect or unpack legacy stores.
+The original ``sgkit.io.bgen.bgen_reader.unpack_variables`` import remains
+supported as a compatibility alias; that module no longer contains a reader.
 
 PLINK
 -----
 
+Use `bio2zarr <https://sgkit-dev.github.io/bio2zarr/>`__ for PLINK-to-Zarr
+conversion and `vcztools <https://github.com/sgkit-dev/vcztools>`_ for export.
+The following sgkit PLINK functions are deprecated.
+
 The :func:`sgkit.io.plink.read_plink` function loads a single PLINK dataset as Dask
 arrays within an :class:`xarray.Dataset` from ``bed``, ``bim``, and ``fam`` files.
 

pyproject.toml

@@ -22,10 +22,10 @@ requires-python = ">=3.11"
 dependencies = [
   "numpy >= 2.3.5; sys_platform == \"darwin\"", # Apple ARM chip error
   "numpy >= 2.0; sys_platform != \"darwin\"",
-  "xarray < 2025.03.1",
+  "xarray >= 2026.9.0",
   "dask[array,dataframe] >= 2022.01.0, <= 2024.8.0",
   "scipy < 1.16",
-  "zarr >= 2.10.0, != 2.11.0, != 2.11.1, != 2.11.2, < 3",
+  "zarr >= 3",
   "numba",
   "fsspec != 2021.6.*",
   "scikit-learn",
@@ -41,7 +41,6 @@ content-type = "text/x-rst"
 
 [project.optional-dependencies]
 plink = ["partd", "bed-reader"]
-bgen = ["rechunker", "cbgen > 1.0.5"]
 
 [dependency-groups]
 test = [
@@ -55,14 +54,14 @@ test = [
   "scikit-allel",
   "statsmodels",
   "msprime>=1.0",
-  "bio2zarr[vcf]; platform_system != \"Windows\"",
+  "bio2zarr[vcf] >= 0.2; platform_system != \"Windows\"",
   "yarl",
   "matplotlib",
   "networkx",
   "aiohttp",
   "requests",
   "graphviz",
-  "sgkit[plink,bgen]",
+  "sgkit[plink]",
 ]
 docs = [
   "graphviz",
@@ -189,14 +188,6 @@ ignore_missing_imports = true
 module = ["sklearn.*"]
 ignore_missing_imports = true
 
-[[tool.mypy.overrides]]
-module = ["cbgen.*"]
-ignore_missing_imports = true
-
-[[tool.mypy.overrides]]
-module = ["rechunker.*"]
-ignore_missing_imports = true
-
 [[tool.mypy.overrides]]
 module = ["bed_reader.*"]
 ignore_missing_imports = true