From 56237d019a6daf27888a35c7f6147512136c4948 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 2 Nov 2026 12:38:51 +0000 Subject: [PATCH] feat(tool-images): build the runner image with other runner ids tool-images/build takes RUNNER_UID and RUNNER_GID, the stock 1001 unless set, and moves every file the stock ids owned to the new ones. Under hostUsers: false a runner pod's hostPath volumes are mounted idmapped, so the node cache and capability store files belong on the node to the runner's own uid, which on some nodes is a person's account. The publish workflow builds a build runner entry with a runner_id under the tag -uid, and names each digest artifact with the architecture before the tag so one tag's pattern never matches another's. tests/runner_build/run.sh checks both sets of ids on amd64 and arm64. --- .github/workflows/publish-tool-images.yml | 21 ++-- .github/workflows/runner-build.yml | 34 +++++++ README.md | 3 +- roles/github_runner_arc/README.md | 4 +- tests/runner_build/run.sh | 61 ++++++++++++ tests/unit/test_publish_tool_images.py | 116 ++++++++++++++++++++++ tool-images/build/Dockerfile | 17 +++- 7 files changed, 245 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/runner-build.yml create mode 100755 tests/runner_build/run.sh create mode 100644 tests/unit/test_publish_tool_images.py diff --git a/.github/workflows/publish-tool-images.yml b/.github/workflows/publish-tool-images.yml index 2765a727..a9f8fb65 100644 --- a/.github/workflows/publish-tool-images.yml +++ b/.github/workflows/publish-tool-images.yml @@ -1,6 +1,6 @@ name: Publish tool images -# Builds the reference images in tool-images/ (the tool-cache and system-library capability images, see the ARC role README's Capabilities section, and the build runner image, the stock runner image with a C toolchain) for linux/amd64 and linux/arm64 and pushes them to this repository owner's GitHub Container Registry namespace, as ghcr.io//github-runner-toolcache-, ghcr.io//github-runner-sysroot- and ghcr.io//github-runner-runner-, each tagged with the version its Dockerfile pins (its ARG _VERSION default; for the build runner image, the runner version it extends). A version tag that already exists is left alone, so a published version never changes under anyone pinning it and a run publishes only the images whose pinned version is new. The job summary lists each pushed image by digest, the reference to pin in a capability or as a runner image. +# Builds the reference images in tool-images/ (the tool-cache and system-library capability images, see the ARC role README's Capabilities section, and the build runner image, the stock runner image with a C toolchain) for linux/amd64 and linux/arm64 and pushes them to this repository owner's GitHub Container Registry namespace, as ghcr.io//github-runner-toolcache-, ghcr.io//github-runner-sysroot- and ghcr.io//github-runner-runner-, each tagged with the version its Dockerfile pins (its ARG _VERSION default; for the build runner image, the runner version it extends). A version tag that already exists is left alone, so a published version never changes under anyone pinning it and a run publishes only the images whose pinned version is new. The job summary lists each pushed image by digest, the reference to pin in a capability or as a runner image. A build runner entry with a runner_id builds the same Dockerfile with RUNNER_UID and RUNNER_GID set to it and tags it -uid: the runner user at that id, for a fleet whose nodes give the stock id (1001) to an account of their own and reserve this one for the runner (tool-images/build/Dockerfile says why the id reaches the nodes). Each entry's digest artifacts are named with the architecture before the tag, so one tag's download pattern never matches another's artifacts. # # Runs only when started by hand or by pushing a tag named tool-images-; the release workflow does not build these images. Each architecture is built natively, on GitHub's amd64 and arm64 hosted runners, and pushed by digest, and a second job joins the two into one multi-architecture tag (https://docs.docker.com/build/ci/github-actions/multi-platform/), because compiling the system libraries under emulation is far slower than natively. @@ -19,7 +19,7 @@ permissions: jobs: build: - name: Build ${{ matrix.image.kind }}-${{ matrix.image.name }} (${{ matrix.platform.arch }}) + name: Build ${{ matrix.image.kind }}-${{ matrix.image.name }}${{ matrix.image.runner_id && format('-uid{0}', matrix.image.runner_id) || '' }} (${{ matrix.platform.arch }}) runs-on: ${{ matrix.platform.runner }} timeout-minutes: 90 permissions: @@ -39,6 +39,7 @@ jobs: - {name: glpk, kind: sysroot} - {name: libmariadb, kind: sysroot} - {name: build, kind: runner} + - {name: build, kind: runner, runner_id: 549} platform: - {arch: amd64, runner: ubuntu-latest} - {arch: arm64, runner: ubuntu-24.04-arm} @@ -51,6 +52,7 @@ jobs: OWNER: ${{ github.repository_owner }} KIND: ${{ matrix.image.kind }} NAME: ${{ matrix.image.name }} + RUNNER_ID: ${{ matrix.image.runner_id }} run: | set -euo pipefail version="$(sed -n 's/^ARG [A-Z_]*_VERSION=//p' "tool-images/${NAME}/Dockerfile")" @@ -59,7 +61,7 @@ jobs: exit 1 fi echo "repository=ghcr.io/${OWNER,,}/github-runner-${KIND}-${NAME}" >>"$GITHUB_OUTPUT" - echo "version=${version}" >>"$GITHUB_OUTPUT" + echo "version=${version}${RUNNER_ID:+-uid${RUNNER_ID}}" >>"$GITHUB_OUTPUT" - uses: docker/login-action@v4 with: @@ -88,6 +90,9 @@ jobs: with: context: tool-images/${{ matrix.image.name }} platforms: linux/${{ matrix.platform.arch }} + build-args: | + ${{ matrix.image.runner_id && format('RUNNER_UID={0}', matrix.image.runner_id) || '' }} + ${{ matrix.image.runner_id && format('RUNNER_GID={0}', matrix.image.runner_id) || '' }} labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} @@ -106,13 +111,13 @@ jobs: - uses: actions/upload-artifact@v7 if: steps.published.outputs.exists == 'false' with: - name: digests-${{ matrix.image.kind }}-${{ matrix.image.name }}-${{ matrix.platform.arch }} + name: digests-${{ matrix.image.kind }}-${{ matrix.image.name }}-${{ matrix.platform.arch }}-${{ steps.image.outputs.version }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 merge: - name: Publish ${{ matrix.image.kind }}-${{ matrix.image.name }} + name: Publish ${{ matrix.image.kind }}-${{ matrix.image.name }}${{ matrix.image.runner_id && format('-uid{0}', matrix.image.runner_id) || '' }} needs: [build] # Each image is joined on its own, so one image's failed build does not hold back the others; the join itself refuses an image missing an architecture. if: ${{ !cancelled() }} @@ -135,6 +140,7 @@ jobs: - {name: glpk, kind: sysroot} - {name: libmariadb, kind: sysroot} - {name: build, kind: runner} + - {name: build, kind: runner, runner_id: 549} steps: - uses: actions/checkout@v7 @@ -144,16 +150,17 @@ jobs: OWNER: ${{ github.repository_owner }} KIND: ${{ matrix.image.kind }} NAME: ${{ matrix.image.name }} + RUNNER_ID: ${{ matrix.image.runner_id }} run: | set -euo pipefail version="$(sed -n 's/^ARG [A-Z_]*_VERSION=//p' "tool-images/${NAME}/Dockerfile")" echo "repository=ghcr.io/${OWNER,,}/github-runner-${KIND}-${NAME}" >>"$GITHUB_OUTPUT" - echo "version=${version}" >>"$GITHUB_OUTPUT" + echo "version=${version}${RUNNER_ID:+-uid${RUNNER_ID}}" >>"$GITHUB_OUTPUT" - uses: actions/download-artifact@v8 with: path: ${{ runner.temp }}/digests - pattern: digests-${{ matrix.image.kind }}-${{ matrix.image.name }}-* + pattern: digests-${{ matrix.image.kind }}-${{ matrix.image.name }}-*-${{ steps.image.outputs.version }} merge-multiple: true - uses: docker/login-action@v4 diff --git a/.github/workflows/runner-build.yml b/.github/workflows/runner-build.yml new file mode 100644 index 00000000..0e0070f9 --- /dev/null +++ b/.github/workflows/runner-build.yml @@ -0,0 +1,34 @@ +name: Runner build image + +# Builds the reference build runner image (tool-images/build) natively on amd64 and arm64, with its default ids and with RUNNER_UID and RUNNER_GID set to another id, and checks the runner user in each (tests/runner_build/run.sh): its ids, no file left with the stock ids once they change, and that it can still write its home, run its own programs, use sudo and reach the docker group. Only on changes that can affect them. Builds nothing it publishes. + +on: + pull_request: + paths: + - tool-images/build/** + - tests/runner_build/** + - .github/workflows/runner-build.yml + workflow_dispatch: + +concurrency: + group: runner-build-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + runner-ids: + name: Runner ids (${{ matrix.platform.arch }}) + runs-on: ${{ matrix.platform.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + platform: + - {arch: amd64, runner: ubuntu-latest} + - {arch: arm64, runner: ubuntu-24.04-arm} + steps: + - uses: actions/checkout@v7 + - name: Build and check the image with each set of ids + run: tests/runner_build/run.sh diff --git a/README.md b/README.md index 9e2db444..359dbb4b 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every jo ``` - **Confirm no disk accumulates across repeated runner jobs:** the Docker host's disk usage (inside the VM, on a host that runs Docker in one) must stay flat across repeated `test-arc-runner.yml` runs. - **Test capabilities:** `tests/capabilities/run.sh` builds the reference Node.js tool image and libpq system-library image, renders a profile with capabilities through the ARC role and the scale-set chart, and runs a pod from the chart's runner template in a throwaway kind cluster to check that the tool reaches the tool cache and `PATH` and the library reaches the sysroot and its search paths; `.github/workflows/capabilities.yml` runs it on pull requests. `tests/system_libraries/run.sh [openmpi|highs|libpq|glpk|libmariadb]...` proves each reference system-library image works relocated: it copies the payload to `/opt/sysroot` as the role's init container does, applies the role's job-started hook, and runs the payload's programs in the stock `actions-runner` image and a program compiled against it in the reference build runner image (`tool-images/build`, the stock image with `build-essential` and `pkg-config`), including a two-process `mpirun` for Open MPI; `.github/workflows/system-libraries.yml` runs it on amd64 and arm64 on pull requests. The images under `tool-images/` build with `docker buildx build --platform linux/arm64,linux/amd64 tool-images/`; `.github/workflows/publish-tool-images.yml` publishes them (see Releases), and the release workflow does not. +- **Test the build runner image's ids:** `tests/runner_build/run.sh` builds `tool-images/build` with its default runner ids and with `RUNNER_UID` and `RUNNER_GID` set to another, and checks the runner user in each; `.github/workflows/runner-build.yml` runs it on amd64 and arm64 for changes to the image. - **Test the registry cache:** `tests/registry_cache/run.sh` installs the cache as the ARC role renders it in a throwaway kind cluster, with Docker Hub, ghcr.io and a private registry behind basic auth, runs a pod from the chart's runner template as the role renders a Docker-in-Docker profile with the cache on, and checks in the cache's access logs that `docker pull` and a `docker-container` buildx builder pulled through it from all three without logging in, then scales the cache to nothing and checks that pulls and builds still work directly; `.github/workflows/registry-cache.yml` runs it on pull requests. - **Lint:** `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (at its `production` profile), and a collection build check on every push and pull request, plus commitlint (on every commit in a pull request, and on the pull request's own title) on pull requests specifically - all gated by one `required-checks` job so a repository ruleset only ever has to require that one check regardless of how many lint jobs exist. hadolint's warning-level pinning advice is visible but non-blocking (`--failure-threshold error`): this repo deliberately floats several images/packages on `latest` rather than pinning. yamllint's config lives in `.yamllint`, relaxing the three default rules that conflict with this project's own conventions (long single-line comments, no leading `---`, GitHub Actions' `on:` key). - **Commit messages:** `.husky/commit-msg` runs commitlint locally (after `npm ci`), the same check CI's commitlint job applies, so a message semantic-release would mishandle is rejected before it is pushed. The machine-wide hooks dispatcher hands off to it; without one, `git config core.hooksPath .husky` enables it. @@ -90,7 +91,7 @@ A release: The `[skip ci]` in the release commit message is deliberate: the image build already happens in this same run using the exact version semantic-release just decided, so there is nothing useful for a second, separately-triggered run of this workflow (or of `ci.yml`) to do against a commit that only changed a version stamp and a changelog - `ci.yml`'s own lint jobs already ran against every commit before it reached `main`. -The reference capability images in `tool-images/` are not part of a release. `.github/workflows/publish-tool-images.yml` builds each one natively on amd64 and arm64 runners and pushes it to `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` or `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins, when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a `tool-images-` tag. It leaves a version tag that already exists alone, and its run summary gives each pushed image's `:@sha256:` reference to pin in a capability or as a runner image. A new GitHub Container Registry package starts private; make it public in its package settings so pods can pull it without a credential. +The reference capability images in `tool-images/` are not part of a release. `.github/workflows/publish-tool-images.yml` builds each one natively on amd64 and arm64 runners and pushes it to `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` or `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins, when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a `tool-images-` tag. It leaves a version tag that already exists alone, and its run summary gives each pushed image's `:@sha256:` reference to pin in a capability or as a runner image. A new GitHub Container Registry package starts private; make it public in its package settings so pods can pull it without a credential. A build runner entry with a `runner_id` publishes the same image with its runner user at that uid and gid, tagged `-uid` (the ARC role README says when a fleet needs one). **What the release push needs:** a deploy key with write access on `ExaDev/github-runner`, whose private key is the `RELEASE_DEPLOY_KEY` repository secret `release.yml` checks out with. If `main` gains a ruleset, add deploy keys as its bypass actor so the release commit can land. Publishing to Ansible Galaxy additionally needs a `GALAXY_API_KEY` repository secret (an API key from `https://galaxy.ansible.com/me/preferences`); without it, the publish step logs that it's skipping and the release still completes. diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 700a358b..b87d404a 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -296,6 +296,8 @@ Open MPI records its install prefix in its libraries and programs, and from `/op The stock `actions-runner` image has no C compiler, `make` or `pkg-config`. Its runner user does have passwordless `sudo`, and the role gives the runner container no security context that takes it away, so a job could `apt-get install` a toolchain, but every job would then download and install one before its first build step, from whatever the distribution's mirror holds that day. A job that compiles against a sysroot library runs instead on the reference build runner image, `ghcr.io/exadev/github-runner-runner-build` (`tool-images/build`), set as the org's `image`: the stock image, pinned by version and digest, with `build-essential` (`gcc`, `g++`, `make` and the C library's headers) and `pkg-config` added, and nothing else. A compiler cannot be a capability, since it is a tree of programs and runtime pieces laid out for one fixed prefix, so it belongs in the image, while the libraries a build links against stay capabilities, each pod getting only the ones its profile names. A job that only runs a payload's programs, or loads its libraries from an interpreter, works on the stock image. `tests/system_libraries/run.sh` checks each reference image both ways: it copies the payload to `/opt/sysroot` as the init container does, renders the profile's hook data through the role's own tasks, runs the role's job-started hook as the runner user, and then, in the stock image, runs the payload's programs (a two-process `mpirun`, the `highs` and `glpsol` commands solving a model, `psql`, `pg_config` and `mariadb_config`) and checks that every program and library resolves its dependencies; and, in the build runner image, compiles a small program against the payload through its pkg-config file (for GLPK, through the search paths alone; for Connector/C, also through `mysql_config` and `-lmysqlclient`) and runs it (for Open MPI, an `MPI_Allreduce` across two processes under `mpirun`). It also checks that `mpirun` fails without `OPAL_PREFIX`, so the declaration above stays necessary. `.github/workflows/system-libraries.yml` runs it for each image on amd64 and arm64 on pull requests that touch the images, the hook or the role's rendering. +The build runner image's runner user is the stock one, uid and gid `1001`, unless it is built with `RUNNER_UID` and `RUNNER_GID`. That matters where runner pods run with `hostUsers: false`: their hostPath volumes, the node cache and the capability store, are mounted idmapped, so a file the runner writes there is owned on the node by the runner's own uid, and on nodes where `1001` belongs to a person's account every cached file is theirs. Such a fleet reserves an id free on all its nodes for an account of its own, builds the image with it (`--build-arg RUNNER_UID= --build-arg RUNNER_GID=`; the publish workflow does so for each build runner entry with a `runner_id`, tagging it `-uid`), and sets `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` to it, so the capabilities' init containers copy as that id too. `tests/runner_build/run.sh` builds the image with the default ids and with another, and checks the runner's ids, that no file keeps the stock ids, and that the runner can still write its home, run its programs, use `sudo` and reach the `docker` group. + A system-library image of your own follows the same pattern: build the library in a stage of the runner image's distribution release with `--prefix=/sysroot` (for CMake, `-DCMAKE_INSTALL_PREFIX=/sysroot -DCMAKE_INSTALL_LIBDIR=lib`), finish the payload, and copy it into `busybox`: ```dockerfile @@ -319,7 +321,7 @@ Then prove it the way `tests/system_libraries/run.sh` does before relying on it: ### Publishing the reference images -`.github/workflows/publish-tool-images.yml` builds every reference image in `tool-images/` natively for `linux/amd64` and `linux/arm64` and pushes it to the repository owner's GitHub Container Registry namespace, as `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` and, for the build runner image, `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins (for the build runner image, the stock runner version it extends, so a new runner release is a new tag). It runs only when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a tag named `tool-images-`; the release workflow does not build these images. A version tag that already exists is left alone, so a published version never changes under anyone using it, and a run publishes only the images whose pinned version is new; to publish a new version, change the Dockerfile's version (and, where it pins one, its checksum) and run it again. The run's summary prints each pushed image as `:@sha256:`, the reference to put in a capability. A newly created GitHub Container Registry package is private until its visibility is changed to public in the package's settings, which pods without a pull credential need. +`.github/workflows/publish-tool-images.yml` builds every reference image in `tool-images/` natively for `linux/amd64` and `linux/arm64` and pushes it to the repository owner's GitHub Container Registry namespace, as `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` and, for the build runner image, `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins (for the build runner image, the stock runner version it extends, so a new runner release is a new tag). It runs only when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a tag named `tool-images-`; the release workflow does not build these images. A version tag that already exists is left alone, so a published version never changes under anyone using it, and a run publishes only the images whose pinned version is new; to publish a new version, change the Dockerfile's version (and, where it pins one, its checksum) and run it again. The run's summary prints each pushed image as `:@sha256:`, the reference to put in a capability. A newly created GitHub Container Registry package is private until its visibility is changed to public in the package's settings, which pods without a pull credential need. A build runner entry with a `runner_id` publishes the same image with its runner user at that uid and gid, tagged `-uid` (see the build runner image's ids above). ### The job-started hook diff --git a/tests/runner_build/run.sh b/tests/runner_build/run.sh new file mode 100755 index 00000000..f9b6c2d4 --- /dev/null +++ b/tests/runner_build/run.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Builds the reference build runner image (tool-images/build) twice for this machine's architecture, once with its default ids and once with RUNNER_UID and RUNNER_GID set to an id the stock image does not use, and checks each: the runner user and its primary group have the ids the build was given, no file in the image is owned by the stock ids once they changed, and the runner can still write its home, run its own scripts, use sudo and reach the docker group, which is what a runner pod needs of the image. +# +# Usage: tests/runner_build/run.sh. Needs Docker; reaches the runner image's registry and its distribution's packages. Leaves the built images (grtest/runner-build:default, grtest/runner-build:other-ids) in place so a rerun reuses the build cache. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +context="${repo_root}/tool-images/build" +stock_uid="$(sed -n 's/^ARG RUNNER_UID=//p' "${context}/Dockerfile")" +stock_gid="$(sed -n 's/^ARG RUNNER_GID=//p' "${context}/Dockerfile")" +# The group the stock image gives the docker socket, which the runner must keep. +docker_gid=123 + +log() { printf '==> %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +# An id below the system range's top that the stock image leaves unused: the first one counting down from 999 that neither its passwd nor its group file names, so the test never depends on a number the base image might take in a later release. +free_id() { + docker run --rm --entrypoint sh "$1" -c 'id=999; while grep -q "^[^:]*:[^:]*:${id}:" /etc/passwd /etc/group; do id=$((id - 1)); done; echo "$id"' +} + +# Runs the checks against one built image: $1 the image, $2 and $3 the uid and gid it must give the runner, $4 and $5 the ids no file may keep (empty when they did not change). +check_image() { + local image=$1 uid=$2 gid=$3 old_uid=$4 old_gid=$5 + docker run --rm --entrypoint bash "$image" -c ' + set -euo pipefail + uid=$1 gid=$2 old_uid=$3 old_gid=$4 docker_gid=$5 + [ "$(id -u)" = "$uid" ] || { echo "runs as uid $(id -u), not $uid" >&2; exit 1; } + [ "$(id -g)" = "$gid" ] || { echo "runs as gid $(id -g), not $gid" >&2; exit 1; } + [ "$(id -un)" = runner ] || { echo "uid $uid is $(id -un), not runner" >&2; exit 1; } + [ "$(getent group runner | cut -d: -f3)" = "$gid" ] || { echo "group runner is not gid $gid" >&2; exit 1; } + id -G | tr " " "\n" | grep -qx "$docker_gid" || { echo "runner lost the docker group ($docker_gid)" >&2; exit 1; } + if [ -n "$old_uid" ]; then + left=$(sudo -n find / -xdev \( -uid "$old_uid" -o -gid "$old_gid" \) | head -n 5) + [ -z "$left" ] || { echo "files still owned by the stock ids: $left" >&2; exit 1; } + fi + [ "$(stat -c %u /home/runner/run.sh)" = "$uid" ] || { echo "run.sh is not owned by the runner" >&2; exit 1; } + touch /home/runner/.write-check && rm /home/runner/.write-check + /home/runner/bin/Runner.Listener --version >/dev/null + sudo -n true + cc --version >/dev/null + ' check "$uid" "$gid" "$old_uid" "$old_gid" "$docker_gid" +} + +log "tool-images/build with its default ids (${stock_uid}:${stock_gid})" +docker build -q -t grtest/runner-build:default "$context" >/dev/null +check_image grtest/runner-build:default "$stock_uid" "$stock_gid" "" "" + +other="$(free_id grtest/runner-build:default)" +[ "$other" != "$stock_uid" ] && [ "$other" != "$stock_gid" ] || fail "no free id below 1000 in the image" +log "tool-images/build with RUNNER_UID and RUNNER_GID ${other}" +docker build -q --build-arg "RUNNER_UID=${other}" --build-arg "RUNNER_GID=${other}" -t grtest/runner-build:other-ids "$context" >/dev/null +check_image grtest/runner-build:other-ids "$other" "$other" "$stock_uid" "$stock_gid" + +# The image refuses an id it already uses rather than giving the runner a shared one: 0 is root's. +log "tool-images/build refuses RUNNER_UID 0" +if docker build -q --build-arg RUNNER_UID=0 -t grtest/runner-build:refused "$context" >/dev/null 2>&1; then + fail "the build accepted RUNNER_UID 0, an id the image already uses" +fi + +log "all checks passed" diff --git a/tests/unit/test_publish_tool_images.py b/tests/unit/test_publish_tool_images.py new file mode 100644 index 00000000..f9c73cb0 --- /dev/null +++ b/tests/unit/test_publish_tool_images.py @@ -0,0 +1,116 @@ +"""Tests that publish-tool-images.yml publishes each matrix entry under a tag of its own: the build and merge matrices list the same entries, a build runner entry with a runner_id passes it to the Dockerfile's RUNNER_UID and RUNNER_GID and suffixes its tag, and one entry's digest artifacts are never matched by another entry's download pattern, which would join the wrong digests under a tag.""" + +from __future__ import annotations + +import fnmatch +import os +import re +import subprocess +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github" / "workflows" / "publish-tool-images.yml" +BUILD_DOCKERFILE = ROOT / "tool-images" / "build" / "Dockerfile" +ARCHITECTURES = ("amd64", "arm64") + + +def load() -> dict[str, Any]: + loaded = yaml.safe_load(WORKFLOW.read_text()) + assert isinstance(loaded, dict) + return loaded + + +def steps(workflow: dict[str, Any], job: str) -> list[dict[str, Any]]: + found = workflow["jobs"][job]["steps"] + assert isinstance(found, list) + return found + + +def step_with(workflow: dict[str, Any], job: str, key: str, value: str) -> dict[str, Any]: + for step in steps(workflow, job): + if step.get(key) == value: + return step + raise AssertionError(f"{job} has no step with {key} {value}") + + +def version_of(workflow: dict[str, Any], job: str, entry: dict[str, Any]) -> str: + """The tag the job's image step gives an entry, by running its shell under the entry's environment.""" + step = step_with(workflow, job, "id", "image") + env = {"OWNER": "example", "KIND": entry["kind"], "NAME": entry["name"], "RUNNER_ID": str(entry.get("runner_id", "")), "PATH": os.environ["PATH"]} + output = ROOT / ".publish-tool-images-test-output" + try: + subprocess.run(["bash", "-c", step["run"]], cwd=ROOT, env={**env, "GITHUB_OUTPUT": str(output)}, check=True) + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + finally: + output.unlink(missing_ok=True) + return values["version"] + + +def expression(template: str, entry: dict[str, Any], **extra: str) -> str: + """Evaluates the two expression shapes the workflow uses for names and build arguments: plain ${{ matrix.image. }} and friends, and ${{ matrix.image.runner_id && format('{0}', matrix.image.runner_id) || '' }}.""" + + def one(match: re.Match[str]) -> str: + body = match.group(1).strip() + conditional = re.fullmatch(r"matrix\.image\.runner_id && format\('([^']*)', matrix\.image\.runner_id\) \|\| ''", body) + if conditional: + return conditional.group(1).replace("{0}", str(entry["runner_id"])) if "runner_id" in entry else "" + if body.startswith("matrix.image."): + return str(entry.get(body.removeprefix("matrix.image."), "")) + if body in extra: + return extra[body] + raise AssertionError(f"unexpected expression {body}") + + return re.sub(r"\$\{\{(.*?)\}\}", one, template) + + +class PublishToolImagesTest(unittest.TestCase): + def setUp(self) -> None: + self.workflow = load() + self.build_entries = self.workflow["jobs"]["build"]["strategy"]["matrix"]["image"] + self.merge_entries = self.workflow["jobs"]["merge"]["strategy"]["matrix"]["image"] + + def test_build_and_merge_list_the_same_entries(self) -> None: + self.assertEqual(self.build_entries, self.merge_entries) + + def test_a_runner_id_reaches_the_dockerfile_and_the_tag(self) -> None: + variants = [entry for entry in self.build_entries if "runner_id" in entry] + self.assertTrue(variants, "the matrix publishes no build runner image with a runner id") + dockerfile = BUILD_DOCKERFILE.read_text() + self.assertIn("ARG RUNNER_UID=", dockerfile) + self.assertIn("ARG RUNNER_GID=", dockerfile) + build_args = step_with(self.workflow, "build", "id", "build")["with"]["build-args"] + for entry in variants: + self.assertEqual(entry["name"], "build", "only the build runner image takes a runner id") + arguments = [line for line in expression(build_args, entry).splitlines() if line.strip()] + self.assertEqual(sorted(argument.strip() for argument in arguments), [f"RUNNER_GID={entry['runner_id']}", f"RUNNER_UID={entry['runner_id']}"]) + for job in ("build", "merge"): + self.assertTrue(version_of(self.workflow, job, entry).endswith(f"-uid{entry['runner_id']}")) + for entry in self.build_entries: + if "runner_id" not in entry: + self.assertEqual([line for line in expression(build_args, entry).splitlines() if line.strip()], [], f"{entry} gets build arguments") + + def test_each_tag_downloads_only_its_own_digests(self) -> None: + upload = step_with(self.workflow, "build", "uses", "actions/upload-artifact@v7")["with"]["name"] + download = step_with(self.workflow, "merge", "uses", "actions/download-artifact@v8")["with"]["pattern"] + artifacts: dict[str, tuple[str, str]] = {} + for entry in self.build_entries: + version = version_of(self.workflow, "build", entry) + self.assertEqual(version, version_of(self.workflow, "merge", entry)) + for arch in ARCHITECTURES: + name = expression(upload, entry, **{"matrix.platform.arch": arch, "steps.image.outputs.version": version}) + self.assertNotIn(name, artifacts, f"two entries upload {name}") + artifacts[name] = (f"{entry['kind']}-{entry['name']}", version) + for entry in self.merge_entries: + version = version_of(self.workflow, "merge", entry) + pattern = expression(download, entry, **{"steps.image.outputs.version": version}) + matched = sorted(name for name in artifacts if fnmatch.fnmatchcase(name, pattern)) + self.assertEqual(len(matched), len(ARCHITECTURES), f"{pattern} matches {matched}") + self.assertEqual({artifacts[name] for name in matched}, {(f"{entry['kind']}-{entry['name']}", version)}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tool-images/build/Dockerfile b/tool-images/build/Dockerfile index 24993e68..18c7489e 100644 --- a/tool-images/build/Dockerfile +++ b/tool-images/build/Dockerfile @@ -1,8 +1,8 @@ -# Runner image for jobs that compile native code: GitHub's stock actions-runner image, pinned by version and digest, with a C and C++ toolchain (build-essential: gcc, g++, make and the C library's headers) and pkg-config added, and nothing else. A compiler cannot be a capability: it is a tree of programs, headers and runtime pieces laid out for one fixed prefix, so it belongs in the image the runner container runs, while the libraries a build links against still come as system-library capabilities (the ARC role README's Capabilities section). The image is used as an org's or profile's runner image in place of the stock one; the runner, its user and its hooks are the stock image's, unchanged. +# Runner image for jobs that compile native code: GitHub's stock actions-runner image, pinned by version and digest, with a C and C++ toolchain (build-essential: gcc, g++, make and the C library's headers) and pkg-config added, and nothing else. A compiler cannot be a capability: it is a tree of programs, headers and runtime pieces laid out for one fixed prefix, so it belongs in the image the runner container runs, while the libraries a build links against still come as system-library capabilities (the ARC role README's Capabilities section). The image is used as an org's or profile's runner image in place of the stock one; the runner, its user and its hooks are the stock image's, unchanged, unless RUNNER_UID or RUNNER_GID below gives the runner user other ids. # # The tag is the stock runner version it is built on (ARG RUNNER_VERSION), so moving to a new runner release publishes a new tag; the digest pins exactly which build of that release it extends. # -# Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 -t /: tool-images/build +# Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 -t /: tool-images/build (add --build-arg RUNNER_UID= --build-arg RUNNER_GID= for a runner user with other ids, and tag it -uid) ARG RUNNER_VERSION=2.338.0 ARG RUNNER_DIGEST=sha256:4ffadc0002b2581327e06101fc8c06cd189232baf79fe561fac9caeb76f5e807 FROM ghcr.io/actions/actions-runner:${RUNNER_VERSION}@${RUNNER_DIGEST} @@ -12,4 +12,17 @@ USER root RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential pkg-config \ && rm -rf /var/lib/apt/lists/* +# The runner user's id and its primary group's, the stock image's (1001) unless set. A runner pod that runs with hostUsers: false gets its hostPath volumes (the role's node cache and capability store) mounted idmapped, so a file the runner writes there is owned on the node by this same id; a fleet whose nodes give 1001 to a person's account builds the image with an id free on every node, which the nodes reserve for an account of their own, and sets the role's github_runner_arc_runner_uid and github_runner_arc_runner_gid to it. groupmod and usermod refuse an id the image already uses, and every file the old ids owned (the runner's home and its installation among them) moves to the new ones, so nothing is left owned by an id the runner no longer has. With the stock ids the step changes no file. +ARG RUNNER_UID=1001 +ARG RUNNER_GID=1001 +RUN set -eu; \ + old_uid="$(id -u runner)"; old_gid="$(id -g runner)"; \ + if [ "$RUNNER_GID" != "$old_gid" ]; then \ + groupmod -g "$RUNNER_GID" runner; \ + find / -xdev -gid "$old_gid" -exec chgrp -h "$RUNNER_GID" {} +; \ + fi; \ + if [ "$RUNNER_UID" != "$old_uid" ]; then \ + usermod -u "$RUNNER_UID" runner; \ + find / -xdev -uid "$old_uid" -exec chown -h "$RUNNER_UID" {} +; \ + fi USER runner