From 036b5379ac2e22910b333f2ac085b23b4f631913 Mon Sep 17 00:00:00 2001 From: Wyatt Childers Date: Thu, 15 Oct 2026 21:35:49 +0000 Subject: [PATCH] Move to GitHub hosting for images and add CI jobs to pull requests for benchmarks [] --- .github/workflows/build-and-test-ci.yml | 105 +++++++++++++++++- .github/workflows/documentation.yml | 11 +- .github/workflows/strict-build-check.yml | 8 +- dev_tools/docker/build-base/Dockerfile | 4 +- dev_tools/docker/ci-runner/Dockerfile | 2 +- .../setup-scripts/install-software.sh | 3 + dev_tools/docker/dev-env/Dockerfile | 2 +- dev_tools/docker/legacy-dev-env/Dockerfile | 2 +- dev_tools/pylibs/edgdocker/__init__.py | 13 ++- 9 files changed, 129 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-test-ci.yml b/.github/workflows/build-and-test-ci.yml index 21c0b0f1af..3d5dbdb9f9 100644 --- a/.github/workflows/build-and-test-ci.yml +++ b/.github/workflows/build-and-test-ci.yml @@ -7,7 +7,7 @@ # # edg-docker-test --posture=extensive --non-interactive # -# inside the edgcpp/ci-runner image. Jobs are staged so GitHub can schedule +# inside the ghcr.io/edgcpp/ci-runner image. Jobs are staged so GitHub can schedule # them in parallel, with each test leg waiting only on its own build (and the # shared timestamp, which runs alongside the builds): # @@ -15,9 +15,14 @@ # build-gcc ─────────────┘ ├──► publish (on failure) # prepare ───────────────┬──► test-i686 ──┘ # build-i686 ────────────┘ +# bench (pull requests only; independent) # # A shared run timestamp keeps results under the same tests/runs// # tree; the publish job republishes failure artifacts from all test legs. +# The bench job runs the cachegrind benchmarks the way +# `edg-docker-bench --posture=release` does and uploads them as bench-results +# (c/ and cpp/ runs, for comparison against the checked-in +# benchmarks/baselines/project/release-{c,cpp}). name: Build & Test CI @@ -46,7 +51,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -87,7 +92,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -134,7 +139,7 @@ jobs: - edg_x86_64 - edg_x86_64_cp container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 # Required for setarch ... -R (disable ASLR) used by the test harness. options: --privileged defaults: @@ -202,7 +207,7 @@ jobs: - edg_x86_32 - edg_x86_32_cp container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 # Required for setarch ... -R (disable ASLR) used by the test harness. options: --privileged defaults: @@ -277,6 +282,26 @@ jobs: path: aggregated-test-results merge-multiple: true + # Record which commit was tested so consumers of the artifact (e.g. + # acknowledg-forge) can check out the matching tree. For pull requests + # this is the merge commit, not the PR head. + - name: Record CI metadata + env: + TESTED_SHA: ${{ github.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + python3 - > aggregated-test-results/ci-meta.json <<'EOF' + import json, os + print(json.dumps({ + 'tested_sha': os.environ['TESTED_SHA'], + 'head_sha': os.environ['HEAD_SHA'], + 'run_id': int(os.environ['RUN_ID']), + 'run_attempt': int(os.environ['RUN_ATTEMPT']), + }, indent = 2)) + EOF + - name: Upload elist, elog, and .rt.tar.gz results uses: actions/upload-artifact@v7 with: @@ -284,3 +309,73 @@ jobs: path: aggregated-test-results/ if-no-files-found: warn retention-days: 14 + + bench: + name: Benchmark linux-gcc-release + if: ${{ github.event_name == 'pull_request' }} + # Benchmark infrastructure problems shouldn't fail a pull request's CI. + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 120 + container: + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 + defaults: + run: + shell: edg-container-exec "bash -eo pipefail {0}" + + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Configure linux-gcc-release + env: + EDG_BASE: ${{ github.workspace }}/bases/docker/dev-env/gcc + run: cmake --preset linux-gcc-release + + - name: Build linux-gcc-release + env: + EDG_BASE: ${{ github.workspace }}/bases/docker/dev-env/gcc + run: cmake --build build/gcc-release --target cpfe cpfe-cp + + - name: Run benchmarks + env: + EDG_BASE: ${{ github.workspace }}/bases/docker/dev-env/gcc + EDG_BENCH_RUNS_KEPT: '0' + run: | + # Use the pull request's own benchmark tooling. + export PATH="${GITHUB_WORKSPACE}/dev_tools/bin:${PATH}" + export PYTHONPATH="${GITHUB_WORKSPACE}/dev_tools/pylibs" + mkdir -p bench-results + for run in c:bin/cpfe cpp:bin/cpfe-cp; do + name="${run%%:*}" + binary="${run#*:}" + (cd build/gcc-release && source environment.sh && + edg-bench "${binary}") + latest="$(ls -1d benchmarks/runs/*/ | sort | tail -n 1)" + mv "${latest}" "bench-results/${name}" + done + + - name: Record CI metadata + env: + TESTED_SHA: ${{ github.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + python3 - > bench-results/ci-meta.json <<'EOF' + import json, os + print(json.dumps({ + 'tested_sha': os.environ['TESTED_SHA'], + 'head_sha': os.environ['HEAD_SHA'], + 'run_id': int(os.environ['RUN_ID']), + 'run_attempt': int(os.environ['RUN_ATTEMPT']), + }, indent = 2)) + EOF + + - name: Upload benchmark results + uses: actions/upload-artifact@v7 + with: + name: bench-results + path: bench-results/ + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 79799fd0e2..28db564634 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -6,10 +6,11 @@ # Build the Sphinx documentation in doc/ with warnings treated as errors, so # documentation changes stay publishable. # -# The build runs in the edgcpp/sphinx-env image, which the site workflow -# described below also builds the published documentation with. That image -# installs the packages pinned in doc/requirements.txt, so after changing -# them, rebuild it with build-edg-env sphinx-env and push it to Docker Hub. +# The build runs in the ghcr.io/edgcpp/sphinx-env image, which the site +# workflow described below also builds the published documentation with. +# That image installs the packages pinned in doc/requirements.txt, so after +# changing them, rebuild it with build-edg-env sphinx-env and push it to +# ghcr.io/edgcpp. # # The documentation is published by the edgcpp/edgcpp.github.io site # workflow, not from here: it builds the release tag into @@ -47,7 +48,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 container: - image: docker.io/edgcpp/sphinx-env:latest + image: ghcr.io/edgcpp/sphinx-env:latest steps: - name: Check out repository uses: actions/checkout@v7 diff --git a/.github/workflows/strict-build-check.yml b/.github/workflows/strict-build-check.yml index 5fa897fb45..04817a2dba 100644 --- a/.github/workflows/strict-build-check.yml +++ b/.github/workflows/strict-build-check.yml @@ -151,7 +151,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -226,7 +226,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: docker.io/edgcpp/ci-runner@sha256:73be12d128b8507f5feb9e73c25f48f14e6b041940ef165bce5fda8efe2b605c + image: ghcr.io/edgcpp/ci-runner@sha256:14b3948c2ce12cfa5472e5a6c468393d15426f3a67685cc35d830ea5db2acdf8 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" diff --git a/dev_tools/docker/build-base/Dockerfile b/dev_tools/docker/build-base/Dockerfile index 7b545b4b45..2f835228cb 100644 --- a/dev_tools/docker/build-base/Dockerfile +++ b/dev_tools/docker/build-base/Dockerfile @@ -20,12 +20,12 @@ RUN /edg/setup-scripts/run-all.sh && rm -rf /edg/setup-scripts/** RUN ln -sf /usr/bin/dash /usr/bin/sh # Install native tools. -COPY --from=edgcpp/native-tools:latest \ +COPY --from=ghcr.io/edgcpp/native-tools:latest \ /edg/native-tools \ /edg/native-tools # Install softfloat tools. -COPY --from=edgcpp/softfloat:latest \ +COPY --from=ghcr.io/edgcpp/softfloat:latest \ /edg/softfloat \ /edg/softfloat diff --git a/dev_tools/docker/ci-runner/Dockerfile b/dev_tools/docker/ci-runner/Dockerfile index c737bb1576..fdf8b09d6e 100644 --- a/dev_tools/docker/ci-runner/Dockerfile +++ b/dev_tools/docker/ci-runner/Dockerfile @@ -3,7 +3,7 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -FROM edgcpp/build-base:latest +FROM ghcr.io/edgcpp/build-base:latest RUN mkdir /edg/tools diff --git a/dev_tools/docker/ci-runner/setup-scripts/install-software.sh b/dev_tools/docker/ci-runner/setup-scripts/install-software.sh index be437730a4..3acf4c1560 100755 --- a/dev_tools/docker/ci-runner/setup-scripts/install-software.sh +++ b/dev_tools/docker/ci-runner/setup-scripts/install-software.sh @@ -12,6 +12,9 @@ packages=() # Add required packages for checkout to function packages+=("nodejs") +# Add valgrind (cachegrind) for the CI benchmark job. +packages+=("valgrind") + # Install the core packages (dnf fetches the package database on install, # so forcing an update here, particularly for development is counter # productive to image build time -- nearly doubling it on a MacBook Pro diff --git a/dev_tools/docker/dev-env/Dockerfile b/dev_tools/docker/dev-env/Dockerfile index 0bbbdb1e48..22a35c9702 100644 --- a/dev_tools/docker/dev-env/Dockerfile +++ b/dev_tools/docker/dev-env/Dockerfile @@ -3,7 +3,7 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -FROM edgcpp/build-base:latest +FROM ghcr.io/edgcpp/build-base:latest RUN mkdir /edg/workspace && \ mkdir /edg/opt-software && \ diff --git a/dev_tools/docker/legacy-dev-env/Dockerfile b/dev_tools/docker/legacy-dev-env/Dockerfile index e1db6b7951..114c294e3c 100644 --- a/dev_tools/docker/legacy-dev-env/Dockerfile +++ b/dev_tools/docker/legacy-dev-env/Dockerfile @@ -30,7 +30,7 @@ RUN /edg/setup-scripts/run-all.sh COPY docker-tools/* /edg/docker-tools/ # Install native tools. -COPY --from=edgcpp/legacy-native-tools:latest \ +COPY --from=ghcr.io/edgcpp/legacy-native-tools:latest \ /edg/native-tools \ /edg/native-tools diff --git a/dev_tools/pylibs/edgdocker/__init__.py b/dev_tools/pylibs/edgdocker/__init__.py index 36267a4b48..2107793a45 100644 --- a/dev_tools/pylibs/edgdocker/__init__.py +++ b/dev_tools/pylibs/edgdocker/__init__.py @@ -54,6 +54,14 @@ 'ci-runner' ) +# The registry namespace EDG images are published under (the GitHub +# organization's container registry). +IMAGE_NAMESPACE = 'ghcr.io/edgcpp' + +def get_image_repository(image_name: str) -> str: + '''Return the registry repository an internal image is published to.''' + return f"{IMAGE_NAMESPACE}/{image_name}" + def get_docker_file_dir(image_name: str) -> Path: '''Return the expected directory of the Dockerfile for an image name.''' return _DOCKER_DIR / image_name @@ -86,11 +94,12 @@ def get_latest_image_tag(image_name: str, *, their pinned name If you want the generic latest tag, use respect_pins = False. The default value is controlled by use_image_pinning(), for uses that expect a particular format, the caller should explicitly specify.''' + repository = get_image_repository(image_name) if respect_pins and image_name in IMAGE_PINS: # This is an officially maintained image synchronized between developers. - return f"docker.io/edgcpp/{image_name}@sha256:{IMAGE_PINS[image_name]}" + return f"{repository}@sha256:{IMAGE_PINS[image_name]}" else: - return f"docker.io/edgcpp/{image_name}:latest" + return f"{repository}:latest" def has_image_tagged(image_name: str) -> bool: '''Return TRUE if there is a docker image already build on this system.'''