diff --git a/.github/workflows/build-s3torchconnectorclient.yml b/.github/workflows/build-s3torchconnectorclient.yml new file mode 100644 index 000000000000..a00bc5c8aff1 --- /dev/null +++ b/.github/workflows/build-s3torchconnectorclient.yml @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/awslabs/s3-connector-for-pytorch/blob/v1.5.0/.github/workflows/wheels.yml +name: Build s3torchconnectorclient wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/s3torchconnectorclient.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-s3torchconnectorclient.yml' + - 'docs/packages/s3torchconnectorclient.yaml' + - 'patches/s3torchconnectorclient/**' + push: + branches: [main] + paths: + - '.github/workflows/build-s3torchconnectorclient.yml' + - 'docs/packages/s3torchconnectorclient.yaml' + - 'patches/s3torchconnectorclient/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: s3torchconnectorclient + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build s3torchconnectorclient ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # Upstream also builds cp38-cp311, but s3torchconnector (this client's own test + # dependency) requires torch, which has no riscv64 wheel below cp312 on our registry. + python: ["cp312", "cp313", "cp314"] + + env: + S3TORCHCONNECTORCLIENT_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout s3-connector-for-pytorch v${{ env.S3TORCHCONNECTORCLIENT_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: awslabs/s3-connector-for-pytorch + ref: v${{ env.S3TORCHCONNECTORCLIENT_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Apply patches + run: git apply -v python-wheels/patches/s3torchconnectorclient/${{ env.S3TORCHCONNECTORCLIENT_VERSION }}/*.patch + + # THIRD-PARTY-LICENSES is generated by upstream's own ORT-based CI job + # (generate_third_party_licenses.yml) and is never checked into the repo, so a plain + # checkout has no equivalent to stage. The patch above points license-files at LICENSE.* + # instead; this collects the licence actually shipped by each statically-linked CRT + # library, read from the same mountpoint-s3-crt-sys crate source cargo fetches for the + # real build, rather than re-implementing upstream's ORT scan. + - name: Write the CRT licence collector + run: | + cat > collect-crt-licenses.sh <<'EOF' + #!/bin/bash + set -euo pipefail + project="${1:?usage: collect-crt-licenses.sh }" + cargo fetch --manifest-path "$project/Cargo.toml" + crt_sys_src="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -mindepth 2 -maxdepth 2 -type d -name 'mountpoint-s3-crt-sys-*' | head -1)" + for lib in aws-lc s2n-tls aws-c-common aws-c-cal aws-c-io aws-c-compression aws-c-http aws-c-sdkutils aws-c-auth aws-checksums aws-c-s3; do + lib_license="$(find "$crt_sys_src/crt/$lib" -maxdepth 1 -name 'LICENSE*' | head -1)" + cp "$lib_license" "$project/LICENSE.$lib" + done + EOF + + - name: Build wheel + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: s3torchconnectorclient + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # The image ships no Rust toolchain for the setuptools-rust/pyo3 extension; rustup + # provisions a native riscv64gc-unknown-linux-gnu one (gotcha 10). fuse/fuse-devel and + # clang/clang-devel (libclang for mountpoint-s3-crt-sys's bindgen) match upstream's own + # manylinux_2_28 before-all, minus its gcc-toolset, which this image's own gcc replaces. + CIBW_BEFORE_ALL_LINUX: >- + dnf -y install fuse fuse-devel clang clang-devel && + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + CIBW_BEFORE_BUILD: bash collect-crt-licenses.sh {project}/s3torchconnectorclient + CIBW_ENVIRONMENT_LINUX: >- + PATH="$PATH:$HOME/.cargo/bin" + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Drops the four CI_*-prefixed integration/e2e commands from upstream's own + # test-command list: they need real AWS S3 buckets this CI has no credentials for. + # The client's own unit tests and the connector's own exercise the "mock" S3 client + # feature instead, so neither needs network access. + CIBW_TEST_REQUIRES: ./s3torchconnector[test] + # PyPI's newest Pillow (the client's own test extra) is sdist-only on riscv64, so take + # the registry's wheel. The container does not inherit the runner's CI variable, so set + # it to get Hypothesis' built-in ci profile (no per-example deadline) on slow runners. + CIBW_TEST_ENVIRONMENT: >- + PIP_ONLY_BINARY=pillow + CI=true + CIBW_TEST_SOURCES: s3torchconnectorclient/python/tst s3torchconnector + CIBW_TEST_COMMAND: >- + pytest -v s3torchconnectorclient/python/tst/unit && + pytest -v s3torchconnector/tst/unit --ignore s3torchconnector/tst/unit/lightning --ignore s3torchconnector/tst/unit/dcp + + - name: Check the extension and the vendored licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + expected = {"LICENSE", "NOTICE", "LICENSE.aws-lc", + "LICENSE.s2n-tls", "LICENSE.aws-c-common", "LICENSE.aws-c-cal", + "LICENSE.aws-c-io", "LICENSE.aws-c-compression", "LICENSE.aws-c-http", + "LICENSE.aws-c-sdkutils", "LICENSE.aws-c-auth", "LICENSE.aws-checksums", + "LICENSE.aws-c-s3"} + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("s3torchconnectorclient/_mountpoint_s3_client") + and n.endswith(".so") for n in names), whl + licenses = {n.split(".dist-info/licenses/", 1)[1] for n in names + if ".dist-info/licenses/" in n and not n.endswith("/")} + assert licenses == expected, f"{whl}: {sorted(licenses)}" + print(whl, "ok") + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: s3torchconnectorclient-${{ env.S3TORCHCONNECTORCLIENT_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish s3torchconnectorclient ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: s3torchconnectorclient-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/s3torchconnectorclient.yaml b/docs/packages/s3torchconnectorclient.yaml new file mode 100644 index 000000000000..50994aabaf1f --- /dev/null +++ b/docs/packages/s3torchconnectorclient.yaml @@ -0,0 +1,6 @@ +package-name: s3torchconnectorclient +source-code: https://github.com/awslabs/s3-connector-for-pytorch +license: BSD-3-Clause AND Apache-2.0 +versions: +- version: 1.5.0 + patched: true diff --git a/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch b/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch new file mode 100644 index 000000000000..eb232f6d1c9c --- /dev/null +++ b/patches/s3torchconnectorclient/1.5.0/0001-ship-the-licences-of-the-statically-linked-dependencies.patch @@ -0,0 +1,34 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Tue, 13 Oct 2026 06:00:00 +0000 +Subject: [PATCH] s3torchconnectorclient: ship the licences of the statically + linked dependencies + +The wheel's native extension statically links the AWS Common Runtime +(aws-lc, s2n-tls, aws-c-common, aws-c-cal, aws-c-io, aws-c-compression, +aws-c-http, aws-c-sdkutils, aws-c-auth, aws-checksums, aws-c-s3) via +mountpoint-s3-crt-sys, but the explicit license-files list only ships +this project's own LICENSE/NOTICE: THIRD-PARTY-LICENSES is generated +by upstream's own ORT-based CI job and is not checked into the repo, +so it never exists in a plain checkout. Drop that entry and match +LICENSE.* instead, where the build stages each vendored dependency's +licence text next to pyproject.toml. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; the upstream wheels have the same THIRD-PARTY-LICENSES generation step and gap when built from a plain checkout] +--- + s3torchconnectorclient/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/s3torchconnectorclient/pyproject.toml b/s3torchconnectorclient/pyproject.toml +index 8542d48..2a0cce0 100644 +--- a/s3torchconnectorclient/pyproject.toml ++++ b/s3torchconnectorclient/pyproject.toml +@@ -45,7 +45,7 @@ find = { where = ["python/src"] } + target = "s3torchconnectorclient._mountpoint_s3_client" + + [tool.setuptools] +-license-files = [ "LICENSE", "THIRD-PARTY-LICENSES", "NOTICE"] ++license-files = [ "LICENSE", "NOTICE", "LICENSE.*"] + + [tool.cibuildwheel] + test-requires = ["./s3torchconnector[test,e2e]"]