diff --git a/.github/assets/image-publishing-architecture.png b/.github/assets/image-publishing-architecture.png new file mode 100644 index 0000000..78645b7 Binary files /dev/null and b/.github/assets/image-publishing-architecture.png differ diff --git a/.github/scripts/python_release.py b/.github/scripts/python_release.py new file mode 100644 index 0000000..0a58eb2 --- /dev/null +++ b/.github/scripts/python_release.py @@ -0,0 +1,263 @@ +"""Guards for the single HugeGraph Python release workflow (Python 3.11+).""" + +import argparse +import email.parser +import hashlib +import json +import os +import re +import subprocess +import tarfile +import tomllib +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from pathlib import Path + +SOURCE = "apache/hugegraph-ai" +PACKAGE = "hugegraph-python" +MODULE = "hugegraph-python-client" +TARGETS = { + "testpypi": ("https://test.pypi.org/legacy/", "https://test.pypi.org"), + "pypi": ("https://upload.pypi.org/legacy/", "https://pypi.org"), +} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def output(**values): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as stream: + for key, value in values.items(): + require("\n" not in value and "\r" not in value, "Invalid output") + stream.write(f"{key}={value}\n") + + +def github(endpoint): + return json.loads( + subprocess.check_output(["gh", "api", f"repos/{SOURCE}/{endpoint}"]) + ) + + +def resolve(ref, target): + require(bool(ref) and not any(c in ref for c in "\n\r"), "Invalid source ref") + tag = ref.removeprefix("refs/tags/") + if target == "pypi": + obj = github("git/ref/tags/" + urllib.parse.quote(tag, safe=""))["object"] + while obj["type"] == "tag": + obj = github("git/tags/" + obj["sha"])["object"] + require(obj["type"] == "commit", "Tag must identify a commit") + sha = obj["sha"] + else: + sha = github("commits/" + urllib.parse.quote(ref, safe=""))["sha"] + tag = "" + require(re.fullmatch(r"[0-9a-f]{40}", sha), "Invalid source SHA") + output(source_sha=sha, tag=tag) + + +def metadata(source, target, tag, version): + project = tomllib.loads((source / MODULE / "pyproject.toml").read_text())["project"] + require(project["name"] == PACKAGE, f"Distribution name must be {PACKAGE}") + base = project["version"] + number = r"(?:0|[1-9][0-9]*)" + require( + re.fullmatch(rf"{number}\.{number}\.{number}", base), + "Source version must be x.y.z", + ) + if target == "pypi": + require(not version, "test_version must be empty for PyPI") + require(tag == base, "Tag/version mismatch") + version = base + else: + require( + re.fullmatch(rf"{number}\.{number}\.{number}\.{number}", version), + "TestPyPI requires test_version in x.y.z.n format", + ) + require( + version.rsplit(".", 1)[0] == base, + "Test version must extend the source version", + ) + subprocess.run( + [ + "uv", + "version", + "--project", + str(source / MODULE), + "--frozen", + version, + ], + check=True, + ) + output(version=version) + + +def artifact_metadata(path): + if path.name.endswith(".whl"): + with zipfile.ZipFile(path) as archive: + names = [n for n in archive.namelist() if n.endswith(".dist-info/METADATA")] + require(len(names) == 1, "Wheel must contain exactly one METADATA") + raw = archive.read(names[0]) + else: + with tarfile.open(path, "r:gz") as archive: + members = [ + m + for m in archive.getmembers() + if m.name.count("/") == 1 and m.name.endswith("/PKG-INFO") + ] + require( + len(members) == 1 and members[0].isfile(), + "Sdist must contain one root PKG-INFO", + ) + raw = archive.extractfile(members[0]).read() + msg = email.parser.BytesParser().parsebytes(raw) + require( + len(msg.get_all("Name", [])) == 1 and len(msg.get_all("Version", [])) == 1, + "Ambiguous metadata", + ) + return msg["Name"], msg["Version"] + + +def inventory(dist, version): + # uv creates this hidden cache marker; upload-artifact omits hidden files. + files = sorted( + p for p in dist.iterdir() if p.name not in ("manifest.json", ".gitignore") + ) + require(len(files) == 2, "Expected exactly one wheel and one sdist") + require(sum(p.name.endswith(".whl") for p in files) == 1, "Expected one wheel") + require(sum(p.name.endswith(".tar.gz") for p in files) == 1, "Expected one sdist") + result = {} + for path in files: + require(path.is_file() and not path.is_symlink(), "Expected regular artifact") + require( + re.fullmatch( + r"hugegraph_python-[A-Za-z0-9_.+!-]+\.(whl|tar\.gz)", path.name + ), + "Unexpected filename", + ) + require( + artifact_metadata(path) == (PACKAGE, version), "Artifact metadata mismatch" + ) + result[path.name] = digest(path) + return result + + +def manifest(dist, sha, version): + require(re.fullmatch(r"[0-9a-f]{40}", sha), "Invalid source SHA") + data = { + "source": SOURCE, + "source_sha": sha, + "name": PACKAGE, + "version": version, + "files": inventory(dist, version), + } + path = dist / "manifest.json" + path.write_text(json.dumps(data, sort_keys=True, indent=2) + "\n") + output(manifest_sha=digest(path)) + + +def verify(dist, sha, version, manifest_sha): + path = dist / "manifest.json" + require(path.is_file() and not path.is_symlink(), "Missing regular manifest") + require(digest(path) == manifest_sha, "Manifest hash mismatch") + data = json.loads(path.read_text()) + require( + data + == { + "source": SOURCE, + "source_sha": sha, + "name": PACKAGE, + "version": version, + "files": inventory(dist, version), + }, + "Manifest contents mismatch", + ) + return data["files"] + + +def remote_files(target, version): + url = f"{TARGETS[target][1]}/pypi/{PACKAGE}/{urllib.parse.quote(version, safe='')}/json" + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as error: + if error.code == 404: + return {} + raise + result = {} + for item in data["urls"]: + name, sha = item["filename"], item["digests"]["sha256"] + require(re.fullmatch(r"[0-9a-f]{64}", sha), "Missing/invalid remote SHA-256") + require(name not in result, "Duplicate remote filename") + result[name] = sha + return result + + +def publish(dist, sha, version, manifest_sha, target): + files = verify(dist, sha, version, manifest_sha) + remote = remote_files(target, version) + # Finish the entire preflight before starting uv, including partial retries. + unexpected = sorted(remote.keys() - files.keys()) + require(not unexpected, f"Unexpected remote artifacts: {', '.join(unexpected)}") + for name, checksum in files.items(): + require( + name not in remote or remote[name] == checksum, + f"Remote hash conflict: {name}", + ) + pending = [str((dist / name).resolve()) for name in files if name not in remote] + if not pending: + print("All artifacts already exist with identical SHA-256; nothing to upload.") + return + require(bool(os.environ.get("UV_PUBLISH_TOKEN")), "Missing environment token") + subprocess.run( + [ + "uv", + "publish", + "--no-config", + "--trusted-publishing", + "never", + "--publish-url", + TARGETS[target][0], + "--check-url", + TARGETS[target][1] + "/simple/", + *pending, + ], + check=True, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "command", choices=("resolve", "metadata", "manifest", "verify", "publish") + ) + parser.add_argument("--target", choices=TARGETS, default="testpypi") + parser.add_argument("--source-ref", default="main") + parser.add_argument("--source", type=Path, default=Path("source")) + parser.add_argument("--tag", default="") + parser.add_argument("--dist", type=Path, default=Path("dist")) + parser.add_argument("--sha", default="") + parser.add_argument("--version", default="") + parser.add_argument("--test-version", default="") + parser.add_argument("--manifest-sha", default="") + args = parser.parse_args() + if args.command == "resolve": + resolve(args.source_ref, args.target) + elif args.command == "metadata": + metadata(args.source, args.target, args.tag, args.test_version) + elif args.command == "manifest": + manifest(args.dist, args.sha, args.version) + elif args.command == "verify": + verify(args.dist, args.sha, args.version, args.manifest_sha) + else: + publish(args.dist, args.sha, args.version, args.manifest_sha, args.target) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish_python.yml b/.github/workflows/publish_python.yml new file mode 100644 index 0000000..020907d --- /dev/null +++ b/.github/workflows/publish_python.yml @@ -0,0 +1,147 @@ +name: Publish HugeGraph Python + +on: + workflow_dispatch: + inputs: + source_ref: + description: 'apache/hugegraph-ai ref (PyPI requires a version tag)' + required: true + default: main + type: string + target: + description: Package index and GitHub environment + required: true + type: choice + default: testpypi + options: [testpypi, pypi] + test_version: + description: 'Required for TestPyPI: x.y.z.n (e.g. 1.7.0.1). Leave empty for PyPI.' + required: false + type: string + +permissions: + contents: read + +env: + UV_VERSION: '0.12.18' + +jobs: + build-check: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + source_sha: ${{ steps.resolve.outputs.source_sha }} + version: ${{ steps.metadata.outputs.version }} + manifest_sha: ${{ steps.manifest.outputs.manifest_sha }} + artifact_id: ${{ steps.upload.outputs.artifact-id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + - name: Resolve immutable source + id: resolve + env: + GH_TOKEN: ${{ github.token }} + SOURCE_REF: ${{ inputs.source_ref }} + TARGET: ${{ inputs.target }} + run: python .github/scripts/python_release.py resolve --source-ref "$SOURCE_REF" --target "$TARGET" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: apache/hugegraph-ai + ref: ${{ steps.resolve.outputs.source_sha }} + path: source + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: ${{ env.UV_VERSION }} + enable-cache: false + - name: Validate and prepare package version + id: metadata + env: + TARGET: ${{ inputs.target }} + TAG: ${{ steps.resolve.outputs.tag }} + SOURCE_SHA: ${{ steps.resolve.outputs.source_sha }} + TEST_VERSION: ${{ inputs.test_version }} + run: | + test "$(git -C source rev-parse HEAD)" = "$SOURCE_SHA" + python .github/scripts/python_release.py metadata --target "$TARGET" --tag "$TAG" --test-version "$TEST_VERSION" + - name: Build and check distributions + id: manifest + env: + SOURCE_SHA: ${{ steps.resolve.outputs.source_sha }} + VERSION: ${{ steps.metadata.outputs.version }} + run: | + uv build --no-sources source/hugegraph-python-client --out-dir dist + uvx --from twine==7.0.0 twine check --strict dist/* + python .github/scripts/python_release.py manifest --sha "$SOURCE_SHA" --version "$VERSION" + - name: Isolated wheel and sdist smoke and client tests + env: + VERSION: ${{ steps.metadata.outputs.version }} + run: | + uv python install 3.10 3.11 + for version in 3.10 3.11; do + for artifact in "$GITHUB_WORKSPACE"/dist/*.whl "$GITHUB_WORKSPACE"/dist/*.tar.gz; do + scratch=$(mktemp -d) + uv venv --python "$version" "$scratch/venv" + uv pip install --python "$scratch/venv/bin/python" "$artifact" + ( + cd "$scratch" + "$scratch/venv/bin/python" -I -c 'import importlib.metadata as m, os; from pyhugegraph.client import PyHugeClient; assert m.version("hugegraph-python") == os.environ["VERSION"]' + uv pip install --python "$scratch/venv/bin/python" 'pytest==8.4.2' + cp -R "$GITHUB_WORKSPACE/source/hugegraph-python-client/src/tests" "$scratch/tests" + "$scratch/venv/bin/python" -m pytest tests -m 'unit or contract' -v --tb=short + ) + done + done + - name: Verify artifacts unchanged after tests + env: + SOURCE_SHA: ${{ steps.resolve.outputs.source_sha }} + VERSION: ${{ steps.metadata.outputs.version }} + MANIFEST_SHA: ${{ steps.manifest.outputs.manifest_sha }} + run: python .github/scripts/python_release.py verify --sha "$SOURCE_SHA" --version "$VERSION" --manifest-sha "$MANIFEST_SHA" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + id: upload + with: + name: python-release-${{ github.run_id }}-${{ github.run_attempt }} + path: dist/ + if-no-files-found: error + retention-days: 14 + + publish: + needs: build-check + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: ${{ inputs.target }} + concurrency: + group: python-publish-${{ inputs.target }}-${{ needs.build-check.outputs.version }} + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: .github/scripts + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ needs.build-check.outputs.artifact_id }} + merge-multiple: true + path: dist + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: ${{ env.UV_VERSION }} + enable-cache: false + - name: Verify all files, preflight hashes, and publish missing artifacts + env: + SOURCE_SHA: ${{ needs.build-check.outputs.source_sha }} + VERSION: ${{ needs.build-check.outputs.version }} + MANIFEST_SHA: ${{ needs.build-check.outputs.manifest_sha }} + TARGET: ${{ inputs.target }} + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + run: | + python .github/scripts/python_release.py publish --sha "$SOURCE_SHA" \ + --version "$VERSION" --manifest-sha "$MANIFEST_SHA" --target "$TARGET" diff --git a/README.md b/README.md index 57d7422..ba790a5 100644 --- a/README.md +++ b/README.md @@ -1,300 +1,88 @@ # HugeGraph Actions -HugeGraph Actions is the shared CI/CD workspace for HugeGraph repositories. -It mainly hosts GitHub Actions workflows for publishing Docker images, validating releases, and coordinating repository-specific automation. +Shared workflows for publishing HugeGraph Docker images and Python packages, validating releases, and repository automation. -## Core Design +## Image publishing architecture -The image publishing workflows are intentionally split into two layers: +![Latest and release wrappers share standard or PD/Store/Server image workflows; validated images reach Docker Hub only when publishing is enabled.](.github/assets/image-publishing-architecture.png) -```text - Trigger - | - +---------------+----------------+ - | | - scheduled / manual manual release - latest publish publish from branch - | | - v v - publish_latest_*.yml publish_release_*.yml - \ / - \ / - +------------+---------------+ - | - v - reusable workflow implementation - | - +-----------------+----------------------------+ - | | - v v -_publish_image_reusable.yml _publish_pd_store_server_reusable.yml - | | - v v -standard single-image flow pd/store/server specialized flow -``` - -The two publishing modes behave differently: - -- `latest` mode - - scheduled or ad-hoc publish for the current default branch line (master in `apache/hugegraph`) - - skips work when the source hash has not changed - - updates the stored `LAST_*_HASH` variable after a successful publish - -- `release` mode - - manual publish from an explicitly selected source ref - - always publishes when invoked - - uses `image_tag` when provided; standard images otherwise derive `x.y.z` - from the source ref (PD/Store/Server requires an explicit `x.y.z` tag) - -## Multi-Platform Build Performance - -BuildKit exposes automatic platform arguments such as `BUILDPLATFORM` and -`TARGETPLATFORM`. A multi-stage Dockerfile can pin an architecture-independent -build stage to the native builder while leaving the runtime stage on the target -platform: - -```dockerfile -FROM --platform=$BUILDPLATFORM maven:3.9.0-eclipse-temurin-11 AS build -RUN mvn package ... - -FROM eclipse-temurin:11-jre-jammy -COPY --from=build /pkg/dist/ /app/ -``` - -Without the explicit build platform, every unqualified `FROM` defaults to the -requested target platform. An amd64 GitHub runner therefore executes the entire -arm64 Maven, Node, compression, or packaging workload through QEMU. Pinning the -portable build stage prevents emulation while the final JRE/base image remains -architecture correct. - -This is a BuildKit-only feature. Buildx requires Docker Engine 19.03 or newer, -and the automatic platform arguments are documented in the Dockerfile frontend: - -- [Docker multi-platform build strategies](https://docs.docker.com/build/building/multi-platform/) -- [Dockerfile automatic platform arguments](https://docs.docker.com/reference/dockerfile/#automatic-platform-args-in-the-global-scope) - -Use this optimization only when the copied build output is portable or is -explicitly cross-compiled for `TARGETOS` / `TARGETARCH`. Native C/C++, CGO, -JNI, platform-classifier artifacts, and downloaded executables must be audited -and covered by real target-platform smoke tests. When the output is inherently -target-specific, use a native target runner instead of forcing `BUILDPLATFORM`. - -The primary performance gain comes from moving portable build work off QEMU. -Registry cache improvements are additional to the native-build or -cross-compilation gains. Keep measured timings in pull requests or dated CI -reports rather than this design document. - -### Read-Only Branch Validation - -Latest wrappers use two execution policies: - -- default branch (`master`, or `main` for AI): publish images, export registry - caches, create manifests, and update the corresponding `LAST_*_HASH` variable. -- non-default ref with `publish=false`: force validation checks, import existing - caches read-only, build all configured platforms, and skip image pushes, cache - exports, manifests, and hash updates. +Wrappers define when and what to publish; the two reusable workflows implement image building and validation. Each run resolves its source repository/ref to a fixed commit. -Set `publish=true` with an explicit `image_tag` to publish a branch build, for -example `source_repository=hugegraph/hugegraph`, `source_ref=helm-dev`, and -`image_tag=helm-dev`. +## Docker images -This allows an upstream Dockerfile branch to be benchmarked before merge without -changing public images or production cache state. +Thin component wrappers call either [the standard image publisher](.github/workflows/_publish_image_reusable.yml) or [the PD/Store/Server publisher](.github/workflows/_publish_pd_store_server_reusable.yml). -For pd/store/server, a manual `master` run can also set `dry_run=true`. This -forces a fresh exact-master multi-platform build and integration check even when -the source hash is unchanged, while disabling image pushes, cache exports, and -hash updates. +| Mode | Trigger | Source and tag | Unchanged source | +| --- | --- | --- | --- | +| Latest | Scheduled or manual | Default source uses `latest`; other refs can derive a tag or use `image_tag` | Skipped only for the configured default source without an explicit tag | +| Release | Manual | Explicit `source_ref`; standard images can derive the version, PD/Store/Server requires `image_tag=x.y.z` | Always runs | -## Critical Path: PD/Store/Server +Manual latest runs default to validation (`publish=false`): build all configured platforms and read caches without pushing images, exporting caches, or updating `LAST_*_HASH`. Scheduled runs publish automatically. Set `publish=true` to publish manually; use an explicit `image_tag` for branch builds. -`pd/store/server` is the most important publishing flow in this repository and uses a dedicated reusable workflow: -[`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml). +Use `source_repository` to select the component's Apache or HugeGraph repository and `source_ref` for a branch, tag or commit. The resolved source SHA and destination `image_tag` are independent. -One candidate job builds PD, Store, HStore Server, and standalone Server as -amd64/arm64 images and loads both variants into Docker's containerd image store. -Source revisions that provide `docker/bake.hcl` use one shared BuildKit graph: -the native Maven stage runs once, the four target-platform runtime images fan -out in parallel, and one shared registry cache is exported. Older source -revisions keep the serial per-Dockerfile compatibility path. -It starts the upstream `docker/docker-compose.dev.yml` topology with -`pull_policy: never`, and runs a functional graph check before any image is -published. Compatible source revisions that have the same service contract but -only contain `docker/docker-compose.yml` use that legacy file as a fallback. The -check executes the Server image's bundled -`/hugegraph-server/scripts/example.groovy` file, verifies the six-vertex, -six-edge sample graph, then performs separate Gremlin read, create, update, and -delete requests. The final query must return to the original 6V/6E baseline. -The same loaded standalone candidate then passes its smoke test. Docker selects -the local amd64 variants for these checks. Only after all enabled checks succeed -are the already loaded multi-platform final tags pushed; the publishing stage -does not rebuild images and does not create temporary architecture tags. +The standard publisher selects Dockerfiles, build contexts, platforms and optional smoke tests through `build_matrix_json`. Images carry OCI source/revision labels. Successful latest publications can update `LAST_*_HASH`. -The precheck override constrains the three JVMs and Store buffers for a small -CI workload. PD and Store are limited to 1 GiB each, Server to 1.5 GiB, while -heap, direct memory, RocksDB, Raft, and worker queues use a low-memory test -profile. These settings are functional-test limits, not production guidance or -a performance baseline. +### PD/Store/Server validation ```text - source ref - | - v - prepare job - (resolve source SHA, explicit image tag, hash gate) - | - v - build_test_publish_multiarch (one job) - shared Maven build stage - | - +--------------------+----------------------------+ - | pd | store | server-hstore | server-standalone | - +--------------------+----------------------------+ - build and load linux/amd64 + linux/arm64 variants - low-memory compose + bundled graph + Gremlin CRUD - standalone smoke test - push loaded x.y.z (or latest) indexes - | - v - update_latest_hash (latest mode only, optional) +Resolve source SHA and tag + → Build/load PD, Store, HStore Server and standalone Server (amd64 + arm64) + → Run local compose, example graph and Gremlin CRUD checks + → Smoke-test standalone Server + → Push the same multi-platform candidates + → Update latest hash, when enabled ``` -Tag behavior: +Sources with `docker/bake.hcl` share one native Maven build and registry cache (`hugegraph/hugegraph:shared-`); older sources use the per-Dockerfile path. Compose uses `docker/docker-compose.dev.yml` with `pull_policy: never`, or the compatible `docker/docker-compose.yml` fallback. -- Final tags contain both `linux/amd64` and `linux/arm64` variants. -- No temporary `*-amd64` or `*-arm64` tags are created. -- Failed builds or functional checks stop the job before any candidate is pushed. +The precheck imports the bundled `example.groovy`, verifies its 6 vertices/6 edges, and exercises Gremlin CRUD before returning to that baseline. Runtime checks use the locally loaded amd64 images. Only successful candidates are pushed, without rebuilding or temporary architecture tags. -Execution note: +The runner uses 1 PD + 1 Store + 1 Server, with 1 GiB each for PD/Store and 1.5 GiB for Server. These are CI limits, not production sizing. A full 3+3+3 topology requires a larger runner or another validated test setup. -- All four current Dockerfiles use `FROM --platform=$BUILDPLATFORM` for their - portable build stages. The x86 runner therefore performs Maven work natively - and limits QEMU to ARM target-image runtime steps. -- Compatible source revisions use `docker/bake.hcl` to deduplicate that native - Maven stage and export it once as `hugegraph/hugegraph:shared-`. -- The single job shares checkout, Docker, QEMU, Buildx, and login setup. Dry-runs - read existing caches but never export new ones. +### Multi-platform builds -## Why The Wrappers Stay Split +Use `FROM --platform=$BUILDPLATFORM` for portable build stages so Maven/Node packaging runs natively; leave runtime stages on the target platform. Native libraries, JNI/CGO and architecture-specific downloads need cross-compilation or native target runners plus runtime tests. See [Docker's build strategies](https://docs.docker.com/build/building/multi-platform/) and [platform arguments](https://docs.docker.com/reference/dockerfile/#automatic-platform-args-in-the-global-scope). -Although the `latest` and `release` wrappers look similar, they encode different release semantics. +## Python packages -- `latest` is the automatic path. - - It is scheduled for daily publication and can also be triggered manually. - - It uses the hash gate to avoid republishing unchanged sources. - - It usually targets the main development branch for each repository. +[`publish_python.yml`](.github/workflows/publish_python.yml) publishes `hugegraph-python` from `apache/hugegraph-ai`. -- `release` is the intentional publication path. - - It is triggered manually. - - Its source `source_ref` and destination `image_tag` are independent. - - It should run even if the source is unchanged, because the operator is explicitly asking for a release publication. +| Input | Default | Meaning | +| --- | --- | --- | +| `source_ref` | `main` | Branch, tag or SHA; resolved to an immutable commit | +| `target` | `testpypi` | `testpypi` or `pypi`; production requires a tag exactly matching the package version | +| `test_version` | Empty | Required for TestPyPI, e.g. `1.7.0.1`; must be empty for PyPI | -Most wrappers use [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). +Test versions use `x.y.z.n`, extending the source version with a fourth number only in the CI checkout. Choose a new number when testing changed code; there is no automatic numbering. Production reads `x.y.z` directly from the source metadata and requires a matching tag, with no version override. Four-part versions are a convention for TestPyPI, not Python prerelease markers. -The pd/store/server wrappers use [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml), which adds an integration precheck and single-job multi-platform publication. +Create environments `testpypi` and `pypi`, each containing `PYPI_API_TOKEN`. Restrict `pypi` deployments to the `master` branch of this repository and require maintainer approval with admin bypass disabled; `testpypi` allows branch validation. This restriction applies to the workflow branch, not the source package tag. Only the upload step receives the selected token. -## Reusable Workflow Responsibilities - -Reusable workflows are the real implementation layer. - -`_publish_image_reusable.yml` handles the standard image flow: - -- resolving `latest` vs `release` mode -- checking out the correct source commit -- deriving the image tag -- selecting per-module build settings from `build_matrix_json` -- enabling QEMU and Buildx when needed -- running optional smoke tests -- stamping `org.opencontainers.image.revision` and `org.opencontainers.image.source` with the resolved source commit and repository -- pushing the final image -- updating the latest-hash variable for `latest` mode only - -`_publish_pd_store_server_reusable.yml` handles the pd/store/server flow: - -- shared source SHA resolution and latest hash gate -- build and locally load multi-platform candidates followed by strict low-memory integration precheck for pd/store/server (hstore backend, `hugegraph/server`) -- import of the Server image's bundled `example.groovy` graph and Gremlin CRUD validation -- publication of the loaded amd64/arm64 index directly to the final tag -- independent release source ref and destination image tag inputs -- standalone server smoke test for `hugegraph/hugegraph` - -The current precheck intentionally uses a 1 PD + 1 Store + 1 Server topology so -it fits standard GitHub-hosted runners. A full 3 PD + 3 Store + 3 Server compose -gate remains a TODO for a larger runner or a reliable lower-resource simulation. - -Wrapper workflows provide the common source and publication contract: - -- `source_repository`: source repository in `owner/name` format -- `allowed_source_repositories`: comma-separated trusted repositories accepted by the wrapper -- `source_ref`: source branch, tag, or commit -- `image_tag`: optional image tag; the configured default source uses `latest`, other latest refs derive a tag when omitted, and release mode derives or validates a version -- `publish`: whether to push images and registry caches - -Only the component's Apache and HugeGraph source repositories are accepted by -the built-in wrappers. Manual runs always respect `publish`; scheduled runs -enable it automatically. Latest hash gating is limited to the configured -default source with no explicit `image_tag`. - -Standard wrappers may also pass `build_matrix_json`; the specialized -pd/store/server workflow defines its four image builds directly. Manual latest -dispatches default to validation for every ref; set `publish=true` and provide -`image_tag` to publish a branch build such as `helm-dev`. An explicit -`image_tag=latest` is also allowed for a non-default source. - -## How To Extend - -When adding a new image publishing workflow, follow the same pattern: - -1. Create a thin `publish_latest_*.yml` wrapper if the image needs to be scheduled or hash-gated for automatic publishing. -2. Create a matching `publish_release_*.yml` wrapper if the image also needs manual release publishing. -3. Put shared build behavior into the appropriate reusable workflow instead of duplicating Docker or checkout logic. -4. Put image-specific values in the wrapper via `build_matrix_json`, especially: - - module name - - Dockerfile path - - build context - - image repository name - - platform list - - optional smoke test command - -Use the reusable workflow for behavior, and the wrapper for policy. - -### When To Keep A Special-Case Workflow Separate - -Keep a dedicated workflow file when the publishing flow has materially different behavior, for example: +```mermaid +flowchart LR + S[Source SHA] --> B[Build and test] + B --> A[Wheel + sdist + hash manifest] + A --> P[Verify and publish] + P --> T[TestPyPI · default] + P --> R[PyPI · version tag required] +``` -- integration prechecks before publishing -- multiple images with custom dependency ordering -- different trigger semantics that do not fit the `latest` / `release` split -- legacy workflows that still require bespoke setup +The build job uses uv, checks wheel/sdist with Twine, records their hashes, and runs isolated installation and client tests on Python 3.10/3.11. It verifies the artifacts again after tests. The publish job checks the manifest and remote filenames/hashes before uploading those exact artifacts. Unexpected remote files or conflicting hashes fail; identical files are skipped. For partial uploads, **re-run failed jobs** to reuse the original artifacts. -For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) and [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) use a dedicated reusable workflow for specialized precheck and publish sequencing. +
+Development checks -## Current Workflow Map +Local checks (no upload): -- Standard reusable publish path: - - [`.github/workflows/publish_latest_loader_image.yml`](./.github/workflows/publish_latest_loader_image.yml) - - [`.github/workflows/publish_release_loader_image.yml`](./.github/workflows/publish_release_loader_image.yml) - - [`.github/workflows/publish_latest_hubble_image.yml`](./.github/workflows/publish_latest_hubble_image.yml) - - [`.github/workflows/publish_release_hubble_image.yml`](./.github/workflows/publish_release_hubble_image.yml) - - [`.github/workflows/publish_latest_vermeer_image.yml`](./.github/workflows/publish_latest_vermeer_image.yml) - - [`.github/workflows/publish_release_vermeer_image.yml`](./.github/workflows/publish_release_vermeer_image.yml) - - [`.github/workflows/publish_latest_ai_image.yml`](./.github/workflows/publish_latest_ai_image.yml) - - [`.github/workflows/publish_release_ai_image.yml`](./.github/workflows/publish_release_ai_image.yml) +```bash +uv run --no-project --python 3.11 python -m unittest discover -s tests -p 'test_python_release.py' -v +actionlint .github/workflows/publish_python.yml +``` -- Dedicated reusable publish path: - - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) - - [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) +
-- Other legacy or special-case workflows: - - [`.github/workflows/publish_hugegraph_hubble.yml`](./.github/workflows/publish_hugegraph_hubble.yml) - - [`.github/workflows/publish_computer_image.yml`](./.github/workflows/publish_computer_image.yml) +A new manual workflow must first reach the default branch to register dispatch; subsequent runs can select another workflow branch with `gh workflow run --ref`. -## Practical Notes +## Maintenance -- `latest` workflows typically run on a schedule and accept manual dispatch. -- `release` workflows typically accept only manual dispatch. They use the same - `source_repository` / `source_ref` contract and accept an optional - independent `image_tag`. -- Most image workflows inherit credentials and settings through a reusable workflow. -- If you change shared standard behavior, update `_publish_image_reusable.yml` first. -- If you change pd/store/server behavior, update `_publish_pd_store_server_reusable.yml` first. +Browse the [workflow directory](.github/workflows) for component entry points. Keep trigger policy and component settings in wrappers, shared build behavior in reusable workflows, and preserve the distinct latest/release semantics. Use dedicated workflows for different validation or sequencing requirements. diff --git a/tests/test_python_release.py b/tests/test_python_release.py new file mode 100644 index 0000000..a508890 --- /dev/null +++ b/tests/test_python_release.py @@ -0,0 +1,339 @@ +"""Exercise release failures without credentials, package indexes, or uploads.""" + +import importlib.util +import io +import json +import os +import subprocess +import sys +import tarfile +import tempfile +import unittest +import urllib.error +import zipfile +from pathlib import Path +from unittest.mock import patch + +SCRIPT = Path(__file__).resolve().parents[1] / ".github/scripts/python_release.py" +SPEC = importlib.util.spec_from_file_location("release", SCRIPT) +release = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(release) +SHA = "a" * 40 + + +class ReleaseTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.dist = self.root / "dist" + self.dist.mkdir() + self.wheel = self.dist / "hugegraph_python-1.7.0-py3-none-any.whl" + self.sdist = self.dist / "hugegraph_python-1.7.0.tar.gz" + self.artifacts() + self.env = patch.dict(os.environ, {"GITHUB_OUTPUT": str(self.root / "output")}) + self.env.start() + self.addCleanup(self.env.stop) + release.manifest(self.dist, SHA, "1.7.0") + self.manifest_sha = release.digest(self.dist / "manifest.json") + + def artifacts(self, name="hugegraph-python", version="1.7.0"): + metadata = f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n".encode() + with zipfile.ZipFile(self.wheel, "w") as archive: + archive.writestr("hugegraph_python-1.7.0.dist-info/METADATA", metadata) + with tarfile.open(self.sdist, "w:gz") as archive: + member = tarfile.TarInfo("hugegraph_python-1.7.0/PKG-INFO") + member.size = len(metadata) + archive.addfile(member, io.BytesIO(metadata)) + + def publish(self): + release.publish(self.dist, SHA, "1.7.0", self.manifest_sha, "testpypi") + + def test_exact_artifacts_verify(self): + (self.dist / ".gitignore").write_text("*") + self.assertEqual( + len(release.verify(self.dist, SHA, "1.7.0", self.manifest_sha)), 2 + ) + + def test_tampered_manifest_rejected_before_network(self): + (self.dist / "manifest.json").write_text("{}") + with ( + patch.object(release, "remote_files") as remote, + self.assertRaisesRegex(ValueError, "Manifest hash"), + ): + self.publish() + remote.assert_not_called() + + def test_modified_artifact_rejected(self): + with zipfile.ZipFile(self.wheel, "a") as archive: + archive.writestr("changed", "changed") + with self.assertRaisesRegex(ValueError, "Manifest contents"): + self.publish() + + def test_post_test_verification_rejects_changed_build_output(self): + command = [ + sys.executable, + str(SCRIPT), + "verify", + "--dist", + str(self.dist), + "--sha", + SHA, + "--version", + "1.7.0", + "--manifest-sha", + self.manifest_sha, + ] + before = subprocess.run(command, capture_output=True, text=True, check=False) + self.assertEqual(before.returncode, 0, before.stderr) + # Simulate test code replacing bytes while retaining valid package metadata. + with zipfile.ZipFile(self.wheel, "a") as archive: + archive.writestr("injected.py", "raise RuntimeError('changed artifact')") + after = subprocess.run(command, capture_output=True, text=True, check=False) + self.assertNotEqual(after.returncode, 0) + self.assertIn("Manifest contents mismatch", after.stderr) + + def test_source_or_version_mismatch_rejected(self): + for sha, version in [("b" * 40, "1.7.0"), (SHA, "1.8.0")]: + with self.subTest(sha=sha, version=version), self.assertRaises(ValueError): + release.verify(self.dist, sha, version, self.manifest_sha) + + def test_wrong_distribution_or_version_rejected(self): + for name, version in [ + ("hugegraph-python-client", "1.7.0"), + ("hugegraph-python", "1.8.0"), + ]: + self.artifacts(name, version) + with ( + self.subTest(name=name, version=version), + self.assertRaisesRegex(ValueError, "metadata"), + ): + release.manifest(self.dist, SHA, "1.7.0") + + def test_extra_missing_and_symlink_files_rejected(self): + extra = self.dist / "extra.txt" + extra.write_text("extra") + with self.assertRaises(ValueError): + self.publish() + extra.unlink() + self.wheel.unlink() + with self.assertRaises(ValueError): + self.publish() + self.wheel.symlink_to(self.sdist) + with self.assertRaisesRegex(ValueError, "regular"): + self.publish() + + def test_conflict_in_last_file_prevents_all_uploads(self): + # Wheel is missing; conflicting sdist sorts last. No partial upload allowed. + with ( + patch.object( + release, "remote_files", return_value={self.sdist.name: "b" * 64} + ), + patch.object(release.subprocess, "run") as upload, + self.assertRaisesRegex(ValueError, "Remote hash conflict"), + ): + self.publish() + upload.assert_not_called() + + def test_identical_retry_is_noop_without_token(self): + files = release.inventory(self.dist, "1.7.0") + with ( + patch.object(release, "remote_files", return_value=files), + patch.object(release.subprocess, "run") as upload, + patch.dict(os.environ, {}, clear=True), + ): + self.publish() + upload.assert_not_called() + + def test_unexpected_remote_file_prevents_upload_and_noop(self): + files = release.inventory(self.dist, "1.7.0") + extra = "hugegraph_python-1.7.0-cp310-cp310-manylinux_2_17_x86_64.whl" + for remote in ({extra: "b" * 64}, {**files, extra: "b" * 64}): + with ( + self.subTest(remote=remote), + patch.object(release, "remote_files", return_value=remote), + patch.object(release.subprocess, "run") as upload, + self.assertRaisesRegex(ValueError, "Unexpected remote artifacts"), + ): + self.publish() + upload.assert_not_called() + + def test_partial_retry_uploads_only_missing_file(self): + with ( + patch.object( + release, + "remote_files", + return_value={self.wheel.name: release.digest(self.wheel)}, + ), + patch.object(release.subprocess, "run") as upload, + patch.dict(os.environ, {"UV_PUBLISH_TOKEN": "test-placeholder"}), + ): + self.publish() + self.assertEqual( + upload.call_args.args[0], + [ + "uv", + "publish", + "--no-config", + "--trusted-publishing", + "never", + "--publish-url", + "https://test.pypi.org/legacy/", + "--check-url", + "https://test.pypi.org/simple/", + str(self.sdist.resolve()), + ], + ) + + def test_missing_token_fails_before_upload(self): + with ( + patch.object(release, "remote_files", return_value={}), + patch.object(release.subprocess, "run") as upload, + patch.dict(os.environ, {}, clear=True), + self.assertRaisesRegex(ValueError, "Missing environment token"), + ): + self.publish() + upload.assert_not_called() + + def test_upload_failure_propagates_without_retry(self): + with ( + patch.object(release, "remote_files", return_value={}), + patch.object( + release.subprocess, + "run", + side_effect=subprocess.CalledProcessError(1, "uv"), + ) as upload, + patch.dict(os.environ, {"UV_PUBLISH_TOKEN": "test-placeholder"}), + self.assertRaises(subprocess.CalledProcessError), + ): + self.publish() + self.assertEqual(upload.call_count, 1) + + def test_only_404_means_missing_release(self): + for status in (404, 403, 429, 500): + error = urllib.error.HTTPError( + "https://test.pypi.org", status, "error", {}, None + ) + with patch.object(release.urllib.request, "urlopen", side_effect=error): + if status == 404: + self.assertEqual(release.remote_files("testpypi", "1.7.0"), {}) + else: + with self.assertRaises(urllib.error.HTTPError): + release.remote_files("testpypi", "1.7.0") + + def test_remote_hash_required(self): + body = io.BytesIO( + json.dumps( + {"urls": [{"filename": self.wheel.name, "digests": {"sha256": ""}}]} + ).encode() + ) + with ( + patch.object(release.urllib.request, "urlopen", return_value=body), + self.assertRaisesRegex(ValueError, "remote SHA-256"), + ): + release.remote_files("pypi", "1.7.0") + + def test_annotated_and_lightweight_tag_resolution(self): + for annotated in (False, True): + responses = [{"object": {"type": "commit", "sha": SHA}}] + if annotated: + responses.insert(0, {"object": {"type": "tag", "sha": "b" * 40}}) + with patch.object(release, "github", side_effect=responses) as github: + release.resolve("refs/tags/1.7.0", "pypi") + self.assertEqual(github.call_args_list[0].args, ("git/ref/tags/1.7.0",)) + + def test_official_missing_tag_cannot_fall_back_to_branch(self): + with ( + patch.object( + release, "github", side_effect=subprocess.CalledProcessError(1, "gh") + ), + self.assertRaises(subprocess.CalledProcessError), + ): + release.resolve("main", "pypi") + + def test_branch_resolves_exact_commit(self): + with patch.object(release, "github", return_value={"sha": SHA}) as github: + release.resolve("refs/heads/cx-python-release", "testpypi") + github.assert_called_once_with("commits/refs%2Fheads%2Fcx-python-release") + + def test_metadata_guards_before_build(self): + module = self.root / release.MODULE + module.mkdir() + project = module / "pyproject.toml" + project.write_text( + '[project]\nname="hugegraph-python-client"\nversion="1.7.0"\n' + ) + with self.assertRaisesRegex(ValueError, "Distribution name"): + release.metadata(self.root, "testpypi", "", "1.7.0.1") + project.write_text('[project]\nname="hugegraph-python"\nversion="1.7.0"\n') + for tag in ("main", "client-v1.7.0", "v1.7.0", "", "1.7.0-rc1"): + with ( + self.subTest(tag=tag), + self.assertRaisesRegex(ValueError, "Tag/version"), + ): + release.metadata(self.root, "pypi", tag, "") + with patch.object(release.subprocess, "run") as update: + release.metadata(self.root, "pypi", "1.7.0", "") + update.assert_not_called() + self.assertIn("version=1.7.0", (self.root / "output").read_text()) + + def test_explicit_version_rules(self): + module = self.root / release.MODULE + module.mkdir() + (module / "pyproject.toml").write_text( + '[project]\nname="hugegraph-python"\nversion="1.7.0"\n' + ) + invalid = [ + ("testpypi", ""), + ("testpypi", "1.7.0"), + ("testpypi", "1.7.0rc1"), + ("testpypi", "1.7.0.dev1"), + ("testpypi", "1.7.0.01"), + ("testpypi", "1.7.0.1.2"), + ("testpypi", "1.8.0.1"), + ("pypi", "1.7.0.1"), + ("pypi", "1.7.0"), + ("pypi", "1.7.0rc1"), + ("pypi", "1.8.0"), + ] + for target, version in invalid: + with ( + self.subTest(target=target, version=version), + patch.object(release.subprocess, "run") as update, + self.assertRaises(ValueError), + ): + release.metadata(self.root, target, "1.7.0", version) + update.assert_not_called() + with patch.object(release.subprocess, "run") as update: + release.metadata(self.root, "testpypi", "", "1.7.0.12") + update.assert_called_once_with( + [ + "uv", + "version", + "--project", + str(module), + "--frozen", + "1.7.0.12", + ], + check=True, + ) + self.assertIn("version=1.7.0.12", (self.root / "output").read_text()) + + def test_source_version_must_have_three_numeric_parts(self): + module = self.root / release.MODULE + module.mkdir() + for version in ("1.7", "1.7.0.1", "1.7.0rc1", "1.7.0.dev1", "01.7.0"): + (module / "pyproject.toml").write_text( + f'[project]\nname="hugegraph-python"\nversion="{version}"\n' + ) + with ( + self.subTest(version=version), + patch.object(release.subprocess, "run") as update, + self.assertRaisesRegex(ValueError, "Source version must be"), + ): + release.metadata(self.root, "pypi", version, "") + update.assert_not_called() + + +if __name__ == "__main__": + unittest.main()