From a764a7429537b07bfcb4371e0ab2ab347a9e2831 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 13 Oct 2026 02:48:46 +0000 Subject: [PATCH 1/3] s3torchconnectorclient: Add version 1.5.0 --- .../build-s3torchconnectorclient.yml | 168 ++++++++++++++++++ docs/packages/s3torchconnectorclient.yaml | 6 + ...f-the-statically-linked-dependencies.patch | 34 ++++ 3 files changed, 208 insertions(+) create mode 100644 .github/workflows/build-s3torchconnectorclient.yml create mode 100644 docs/packages/s3torchconnectorclient.yaml create mode 100644 patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch diff --git a/.github/workflows/build-s3torchconnectorclient.yml b/.github/workflows/build-s3torchconnectorclient.yml new file mode 100644 index 000000000000..e2c48cf3e50f --- /dev/null +++ b/.github/workflows/build-s3torchconnectorclient.yml @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/awslabs/s3-connector-for-pytorch/blob/v1.5.0/.github/workflows/wheels.yml +name: Build s3torchconnectorclient wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/s3torchconnectorclient.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-s3torchconnectorclient.yml' + - 'docs/packages/s3torchconnectorclient.yaml' + - 'patches/s3torchconnectorclient/**' + push: + branches: [main] + paths: + - '.github/workflows/build-s3torchconnectorclient.yml' + - 'docs/packages/s3torchconnectorclient.yaml' + - 'patches/s3torchconnectorclient/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: s3torchconnectorclient + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build s3torchconnectorclient ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # Upstream also builds cp38-cp311, but s3torchconnector (this client's own test + # dependency) requires torch, which has no riscv64 wheel below cp312 on our registry. + python: ["cp312", "cp313", "cp314"] + + env: + S3TORCHCONNECTORCLIENT_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout s3-connector-for-pytorch v${{ env.S3TORCHCONNECTORCLIENT_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: awslabs/s3-connector-for-pytorch + ref: v${{ env.S3TORCHCONNECTORCLIENT_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Apply patches + run: git apply -v python-wheels/patches/s3torchconnectorclient/${{ env.S3TORCHCONNECTORCLIENT_VERSION }}/*.patch + + # THIRD-PARTY-LICENSES is generated by upstream's own ORT-based CI job + # (generate_third_party_licenses.yml) and is never checked into the repo, so a plain + # checkout has no equivalent to stage. The patch above points license-files at LICENSE.* + # instead; this collects the licence actually shipped by each statically-linked CRT + # library, read from the same mountpoint-s3-crt-sys crate source cargo fetches for the + # real build, rather than re-implementing upstream's ORT scan. + - name: Write the CRT licence collector + run: | + cat > collect-crt-licenses.sh <<'EOF' + #!/bin/bash + set -euo pipefail + project="${1:?usage: collect-crt-licenses.sh }" + cargo fetch --manifest-path "$project/Cargo.toml" + crt_sys_src="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -maxdepth 1 -type d -name 'mountpoint-s3-crt-sys-*' | head -1)" + cp "$crt_sys_src/LICENSE" "$project/LICENSE.mountpoint-s3-crt-sys" + for lib in aws-lc s2n-tls aws-c-common aws-c-cal aws-c-io aws-c-compression aws-c-http aws-c-sdkutils aws-c-auth aws-checksums aws-c-s3; do + lib_license="$(find "$crt_sys_src/crt/$lib" -maxdepth 1 -name 'LICENSE*' | head -1)" + cp "$lib_license" "$project/LICENSE.$lib" + done + EOF + + - name: Build wheel + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: s3torchconnectorclient + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # The image ships no Rust toolchain for the setuptools-rust/pyo3 extension; rustup + # provisions a native riscv64gc-unknown-linux-gnu one (gotcha 10). fuse/fuse-devel + # match upstream's own before-all exactly, minus its CentOS7-only devtoolset/ + # llvm-toolset packages, which this image's own gcc makes unnecessary. + CIBW_BEFORE_ALL_LINUX: >- + dnf -y install fuse fuse-devel && + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + CIBW_BEFORE_BUILD: bash collect-crt-licenses.sh {project}/s3torchconnectorclient + CIBW_ENVIRONMENT_LINUX: >- + PATH="$PATH:$HOME/.cargo/bin" + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Drops the four CI_*-prefixed integration/e2e commands from upstream's own + # test-command list: they need real AWS S3 buckets this CI has no credentials for. + # The client's own unit tests and the connector's own exercise the "mock" S3 client + # feature instead, so neither needs network access. + CIBW_TEST_REQUIRES: ./s3torchconnector[test] + CIBW_TEST_SOURCES: s3torchconnectorclient/python/tst s3torchconnector + CIBW_TEST_COMMAND: >- + pytest -v s3torchconnectorclient/python/tst/unit && + pytest -v s3torchconnector/tst/unit --ignore s3torchconnector/tst/unit/lightning --ignore s3torchconnector/tst/unit/dcp + + - name: Check the extension and the vendored licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + expected = {"LICENSE", "NOTICE", "LICENSE.mountpoint-s3-crt-sys", "LICENSE.aws-lc", + "LICENSE.s2n-tls", "LICENSE.aws-c-common", "LICENSE.aws-c-cal", + "LICENSE.aws-c-io", "LICENSE.aws-c-compression", "LICENSE.aws-c-http", + "LICENSE.aws-c-sdkutils", "LICENSE.aws-c-auth", "LICENSE.aws-checksums", + "LICENSE.aws-c-s3"} + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("s3torchconnectorclient/_mountpoint_s3_client") + and n.endswith(".so") for n in names), whl + licenses = {n.split(".dist-info/licenses/", 1)[1] for n in names + if ".dist-info/licenses/" in n and not n.endswith("/")} + assert licenses == expected, f"{whl}: {sorted(licenses)}" + print(whl, "ok") + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: s3torchconnectorclient-${{ env.S3TORCHCONNECTORCLIENT_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish s3torchconnectorclient ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: s3torchconnectorclient-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/s3torchconnectorclient.yaml b/docs/packages/s3torchconnectorclient.yaml new file mode 100644 index 000000000000..50994aabaf1f --- /dev/null +++ b/docs/packages/s3torchconnectorclient.yaml @@ -0,0 +1,6 @@ +package-name: s3torchconnectorclient +source-code: https://github.com/awslabs/s3-connector-for-pytorch +license: BSD-3-Clause AND Apache-2.0 +versions: +- version: 1.5.0 + patched: true diff --git a/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch b/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch new file mode 100644 index 000000000000..eb232f6d1c9c --- /dev/null +++ b/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch @@ -0,0 +1,34 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Tue, 13 Oct 2026 06:00:00 +0000 +Subject: [PATCH] s3torchconnectorclient: ship the licences of the statically + linked dependencies + +The wheel's native extension statically links the AWS Common Runtime +(aws-lc, s2n-tls, aws-c-common, aws-c-cal, aws-c-io, aws-c-compression, +aws-c-http, aws-c-sdkutils, aws-c-auth, aws-checksums, aws-c-s3) via +mountpoint-s3-crt-sys, but the explicit license-files list only ships +this project's own LICENSE/NOTICE: THIRD-PARTY-LICENSES is generated +by upstream's own ORT-based CI job and is not checked into the repo, +so it never exists in a plain checkout. Drop that entry and match +LICENSE.* instead, where the build stages each vendored dependency's +licence text next to pyproject.toml. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; the upstream wheels have the same THIRD-PARTY-LICENSES generation step and gap when built from a plain checkout] +--- + s3torchconnectorclient/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/s3torchconnectorclient/pyproject.toml b/s3torchconnectorclient/pyproject.toml +index 8542d48..2a0cce0 100644 +--- a/s3torchconnectorclient/pyproject.toml ++++ b/s3torchconnectorclient/pyproject.toml +@@ -45,7 +45,7 @@ find = { where = ["python/src"] } + target = "s3torchconnectorclient._mountpoint_s3_client" + + [tool.setuptools] +-license-files = [ "LICENSE", "THIRD-PARTY-LICENSES", "NOTICE"] ++license-files = [ "LICENSE", "NOTICE", "LICENSE.*"] + + [tool.cibuildwheel] + test-requires = ["./s3torchconnector[test,e2e]"] From 10f9a8e43b2a6e7ae3bb95a19e2f2886732d0544 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 22 Oct 2026 10:56:09 +0000 Subject: [PATCH 2/3] s3torchconnectorclient: fix CRT licence collection and install libclang The cargo registry keeps crate sources one level below registry/src (index.crates.io-/), so the -maxdepth 1 lookup found nothing and the copy failed on '/LICENSE'. The mountpoint-s3-crt-sys crate also ships no top-level LICENSE, only one per vendored CRT library, so drop that copy. mountpoint-s3-crt-sys generates its bindings with bindgen at build time, which needs libclang: install clang/clang-devel as upstream's own manylinux_2_28 before-all does. --- .github/workflows/build-s3torchconnectorclient.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-s3torchconnectorclient.yml b/.github/workflows/build-s3torchconnectorclient.yml index e2c48cf3e50f..c4fc6d35c640 100644 --- a/.github/workflows/build-s3torchconnectorclient.yml +++ b/.github/workflows/build-s3torchconnectorclient.yml @@ -88,8 +88,7 @@ jobs: set -euo pipefail project="${1:?usage: collect-crt-licenses.sh }" cargo fetch --manifest-path "$project/Cargo.toml" - crt_sys_src="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -maxdepth 1 -type d -name 'mountpoint-s3-crt-sys-*' | head -1)" - cp "$crt_sys_src/LICENSE" "$project/LICENSE.mountpoint-s3-crt-sys" + crt_sys_src="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -mindepth 2 -maxdepth 2 -type d -name 'mountpoint-s3-crt-sys-*' | head -1)" for lib in aws-lc s2n-tls aws-c-common aws-c-cal aws-c-io aws-c-compression aws-c-http aws-c-sdkutils aws-c-auth aws-checksums aws-c-s3; do lib_license="$(find "$crt_sys_src/crt/$lib" -maxdepth 1 -name 'LICENSE*' | head -1)" cp "$lib_license" "$project/LICENSE.$lib" @@ -105,11 +104,11 @@ jobs: env: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} # The image ships no Rust toolchain for the setuptools-rust/pyo3 extension; rustup - # provisions a native riscv64gc-unknown-linux-gnu one (gotcha 10). fuse/fuse-devel - # match upstream's own before-all exactly, minus its CentOS7-only devtoolset/ - # llvm-toolset packages, which this image's own gcc makes unnecessary. + # provisions a native riscv64gc-unknown-linux-gnu one (gotcha 10). fuse/fuse-devel and + # clang/clang-devel (libclang for mountpoint-s3-crt-sys's bindgen) match upstream's own + # manylinux_2_28 before-all, minus its gcc-toolset, which this image's own gcc replaces. CIBW_BEFORE_ALL_LINUX: >- - dnf -y install fuse fuse-devel && + dnf -y install fuse fuse-devel clang clang-devel && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y CIBW_BEFORE_BUILD: bash collect-crt-licenses.sh {project}/s3torchconnectorclient CIBW_ENVIRONMENT_LINUX: >- @@ -129,7 +128,7 @@ jobs: run: | python3 - wheelhouse/*.whl <<'EOF' import sys, zipfile - expected = {"LICENSE", "NOTICE", "LICENSE.mountpoint-s3-crt-sys", "LICENSE.aws-lc", + expected = {"LICENSE", "NOTICE", "LICENSE.aws-lc", "LICENSE.s2n-tls", "LICENSE.aws-c-common", "LICENSE.aws-c-cal", "LICENSE.aws-c-io", "LICENSE.aws-c-compression", "LICENSE.aws-c-http", "LICENSE.aws-c-sdkutils", "LICENSE.aws-c-auth", "LICENSE.aws-checksums", From 40d256ca5a09b9d56dcfbf1f20afaeaa611b6e0f Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 23 Oct 2026 15:50:24 +0000 Subject: [PATCH 3/3] s3torchconnectorclient: take Pillow from the registry and drop Hypothesis deadlines in tests The wheel built and repaired on every interpreter, but installing its own [test] extra failed: PyPI's newest Pillow (13.0.0) has no riscv64 wheel, so pip picked its sdist over the registry's 12.3.0 wheel and the build died on the missing libjpeg headers. PIP_ONLY_BINARY=pillow makes the registry wheel the only candidate. cibuildwheel's container does not inherit the runner's CI variable, so Hypothesis keeps its default 200ms per-example deadline, which the connector's unit suite already exceeds intermittently on x86_64 (test_s3_client_custom_config). CI=true selects Hypothesis' built-in ci profile, which drops the deadline and the too_slow health check. --- .github/workflows/build-s3torchconnectorclient.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-s3torchconnectorclient.yml b/.github/workflows/build-s3torchconnectorclient.yml index c4fc6d35c640..a00bc5c8aff1 100644 --- a/.github/workflows/build-s3torchconnectorclient.yml +++ b/.github/workflows/build-s3torchconnectorclient.yml @@ -119,6 +119,12 @@ jobs: # The client's own unit tests and the connector's own exercise the "mock" S3 client # feature instead, so neither needs network access. CIBW_TEST_REQUIRES: ./s3torchconnector[test] + # PyPI's newest Pillow (the client's own test extra) is sdist-only on riscv64, so take + # the registry's wheel. The container does not inherit the runner's CI variable, so set + # it to get Hypothesis' built-in ci profile (no per-example deadline) on slow runners. + CIBW_TEST_ENVIRONMENT: >- + PIP_ONLY_BINARY=pillow + CI=true CIBW_TEST_SOURCES: s3torchconnectorclient/python/tst s3torchconnector CIBW_TEST_COMMAND: >- pytest -v s3torchconnectorclient/python/tst/unit &&