|
| 1 | +# GitLab CI for libre-devops-helpers. |
| 2 | +# |
| 3 | +# The same checks as GitHub's Lint and Test (.github/workflows/ci.yml), the container |
| 4 | +# images built, smoke tested and scanned as the Container workflow does, and for a release |
| 5 | +# tag, the release published to this project's own registries: |
| 6 | +# |
| 7 | +# - the wheel and sdist to its PyPI package registry; |
| 8 | +# - both images to its container registry, with the tags they have on GHCR; |
| 9 | +# - a GitLab release that links to both. |
| 10 | +# |
| 11 | +# It never publishes anywhere else: PyPI and GHCR are GitHub's to publish to. On gitlab.com |
| 12 | +# it runs on the mirror of the GitHub repository (docs/development.md, "The GitLab mirror"), |
| 13 | +# to be proven before the code moves to a GitLab of its own. What depends on where it runs |
| 14 | +# is a variable, to set at the project or group level there: |
| 15 | +# |
| 16 | +# UV_IMAGE_REPOSITORY, PODMAN_IMAGE, GITLEAKS_IMAGE the images, e.g. through a proxy |
| 17 | +# UV_DEFAULT_INDEX a package index proxy (uv reads it) |
| 18 | +# |
| 19 | +# tests/project/test_gitlab_ci.py keeps the versions here in step with the GitHub workflows |
| 20 | +# and the Containerfile, which Dependabot updates; it cannot update this file. |
| 21 | + |
| 22 | +workflow: |
| 23 | + # The default branch, release tags, and pipelines started by hand or on a schedule. Not |
| 24 | + # every mirrored branch (Dependabot's, the badges): GitHub has checked those already, and |
| 25 | + # each would spend CI minutes. |
| 26 | + rules: |
| 27 | + - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/ |
| 28 | + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH |
| 29 | + - if: $CI_PIPELINE_SOURCE =~ /^(web|api|schedule)$/ |
| 30 | + # A newer push to the same branch cancels the checks still running for an older one. |
| 31 | + auto_cancel: |
| 32 | + on_new_commit: interruptible |
| 33 | + |
| 34 | +stages: [check, build, container, publish] |
| 35 | + |
| 36 | +variables: |
| 37 | + # The uv image is the Containerfile's uv, on the Python each job tests. |
| 38 | + UV_IMAGE_REPOSITORY: ghcr.io/astral-sh/uv |
| 39 | + UV_VERSION: "0.12.18" |
| 40 | + PYTHON: "3.13" |
| 41 | + PODMAN_IMAGE: quay.io/podman/stable:v5.8.7-immutable@sha256:f345d8d4fddea244d3a1ced27bcec82d14cc9e32cb5ead7f5882b0baf59dc8b0 |
| 42 | + GITLEAKS_IMAGE: ghcr.io/gitleaks/gitleaks:v8.30.1@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f |
| 43 | + PIP_AUDIT_VERSION: "2.10.1" |
| 44 | + TRIVY_VERSION: "0.74.0" |
| 45 | + # From the release's checksums file. Bump both together. |
| 46 | + TRIVY_SHA256: 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a |
| 47 | + # The public mirror first; ghcr.io rate limits anonymous database downloads. |
| 48 | + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db,ghcr.io/aquasecurity/trivy-db |
| 49 | + # The image's own Python, never one uv downloads. |
| 50 | + UV_PYTHON_DOWNLOADS: never |
| 51 | + |
| 52 | +default: |
| 53 | + image: ${UV_IMAGE_REPOSITORY}:${UV_VERSION}-python${PYTHON}-trixie-slim |
| 54 | + interruptible: true |
| 55 | + timeout: 20 minutes |
| 56 | + retry: |
| 57 | + max: 1 |
| 58 | + when: [runner_system_failure, scheduler_failure, api_failure] |
| 59 | + before_script: |
| 60 | + # uv then fails, rather than quietly using another Python, if the image lacks this one. |
| 61 | + - export UV_PYTHON="${PYTHON}" |
| 62 | + |
| 63 | +.release-tag: &release-tag |
| 64 | + if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/ |
| 65 | + |
| 66 | +# Checks ------------------------------------------------------------------------------ |
| 67 | + |
| 68 | +# Every commit, so a secret added and later removed is still found. |
| 69 | +secrets: |
| 70 | + stage: check |
| 71 | + image: |
| 72 | + name: $GITLEAKS_IMAGE |
| 73 | + entrypoint: [""] |
| 74 | + variables: |
| 75 | + GIT_DEPTH: "0" |
| 76 | + script: |
| 77 | + - git config --global --add safe.directory "${CI_PROJECT_DIR}" |
| 78 | + - gitleaks git --config .gitleaks.toml --redact --verbose . |
| 79 | + |
| 80 | +lint: |
| 81 | + stage: check |
| 82 | + script: |
| 83 | + # --locked fails the job when uv.lock is out of step with pyproject.toml. |
| 84 | + - uv sync --locked |
| 85 | + # The findings as a Code Quality report for GitLab, then the check that fails the job. |
| 86 | + - uv run ruff check --exit-zero --output-format gitlab --output-file gl-code-quality.json src tests scripts |
| 87 | + - uv run ruff check src tests scripts |
| 88 | + # Check only: formatting is fixed locally with 'just fmt', never in CI. |
| 89 | + - uv run ruff format --check src tests scripts |
| 90 | + - uv run mypy |
| 91 | + artifacts: |
| 92 | + when: always |
| 93 | + reports: |
| 94 | + codequality: gl-code-quality.json |
| 95 | + |
| 96 | +# The locked, hashed dependency tree, runtime and dev, exactly as the jobs install it. |
| 97 | +audit: |
| 98 | + stage: check |
| 99 | + script: |
| 100 | + - uv export --frozen --all-extras --format requirements-txt --no-emit-project --output-file /tmp/requirements.txt |
| 101 | + - uvx "pip-audit==${PIP_AUDIT_VERSION}" --strict --disable-pip --require-hashes --requirement /tmp/requirements.txt |
| 102 | + |
| 103 | +# Every supported Python but the coverage job's. Includes the rebrand test, which renames a |
| 104 | +# copy of the repository and runs its whole suite. |
| 105 | +test: |
| 106 | + stage: check |
| 107 | + parallel: |
| 108 | + matrix: |
| 109 | + - PYTHON: ["3.11", "3.12", "3.14"] |
| 110 | + script: |
| 111 | + - uv sync --locked |
| 112 | + - uv run python --version |
| 113 | + - uv run pytest --junitxml=report.xml |
| 114 | + artifacts: |
| 115 | + when: always |
| 116 | + reports: |
| 117 | + junit: report.xml |
| 118 | + |
| 119 | +# Line and branch coverage, on one Python, so the number is the same on every run. The |
| 120 | +# floor (fail_under) is in pyproject.toml. |
| 121 | +coverage: |
| 122 | + stage: check |
| 123 | + script: |
| 124 | + - uv sync --locked |
| 125 | + - uv run python --version |
| 126 | + - uv run pytest --cov --cov-report=term-missing --cov-report=xml --junitxml=report.xml |
| 127 | + coverage: '/^TOTAL.*\s(\d+(?:\.\d+)?)%$/' |
| 128 | + artifacts: |
| 129 | + when: always |
| 130 | + reports: |
| 131 | + junit: report.xml |
| 132 | + coverage_report: |
| 133 | + coverage_format: cobertura |
| 134 | + path: coverage.xml |
| 135 | + |
| 136 | +# Build ------------------------------------------------------------------------------- |
| 137 | + |
| 138 | +# Built once here; the release publishes these exact files rather than rebuilding. |
| 139 | +build: |
| 140 | + stage: build |
| 141 | + needs: [secrets, lint, audit, test, coverage] |
| 142 | + # None of their artifacts (test reports): an sdist takes in what is in the checkout. |
| 143 | + dependencies: [] |
| 144 | + script: |
| 145 | + - | |
| 146 | + set -euo pipefail |
| 147 | + version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" |
| 148 | + if [ -n "${CI_COMMIT_TAG:-}" ] && [ "v${version}" != "${CI_COMMIT_TAG}" ]; then |
| 149 | + echo "tag ${CI_COMMIT_TAG} does not match pyproject.toml version ${version}" |
| 150 | + exit 1 |
| 151 | + fi |
| 152 | + uv build |
| 153 | + # The wheel installs cleanly, and its entry point resolves outside the source tree. |
| 154 | + uv venv --quiet /tmp/smoke |
| 155 | + uv pip install --quiet --python /tmp/smoke dist/*.whl |
| 156 | + /tmp/smoke/bin/ldo --version |
| 157 | + /tmp/smoke/bin/ldo --help > /dev/null |
| 158 | + # For the jobs after this one: the version, and the tags each image is published with. |
| 159 | + echo "VERSION=${version}" > build.env |
| 160 | + mkdir -p image-tags |
| 161 | + for variant in az tool; do |
| 162 | + python3 scripts/image_tags.py --version "${version}" --variant "${variant}" \ |
| 163 | + --stamp "$(date -u +%Y%m%d).${CI_PIPELINE_IID}" --latest > "image-tags/${variant}.txt" |
| 164 | + done |
| 165 | + artifacts: |
| 166 | + paths: [dist/, image-tags/] |
| 167 | + reports: |
| 168 | + dotenv: build.env |
| 169 | + expire_in: 14 days |
| 170 | + |
| 171 | +# Built, smoke tested and scanned for a release, by hand, and when the image's own files |
| 172 | +# change on the default branch; for a release tag, then pushed to this project's registry. |
| 173 | +# One platform (amd64): an emulated arm64 build would take most of the CI minutes. |
| 174 | +container: |
| 175 | + stage: container |
| 176 | + image: $PODMAN_IMAGE |
| 177 | + needs: [build] |
| 178 | + timeout: 60 minutes |
| 179 | + parallel: |
| 180 | + matrix: |
| 181 | + # The default image. The Azure CLI pins its own dependencies, so only a fixed |
| 182 | + # critical finding fails it (see container/azure-cli/pyproject.toml). |
| 183 | + - VARIANT: az |
| 184 | + GATE: CRITICAL |
| 185 | + # The tool alone is ours to keep clean: any fixed high or critical finding fails. |
| 186 | + - VARIANT: tool |
| 187 | + GATE: HIGH,CRITICAL |
| 188 | + rules: |
| 189 | + - *release-tag |
| 190 | + - if: $CI_PIPELINE_SOURCE =~ /^(web|api|schedule)$/ |
| 191 | + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH |
| 192 | + changes: [Containerfile, container/**/*, .gitlab-ci.yml] |
| 193 | + script: |
| 194 | + - | |
| 195 | + set -euo pipefail |
| 196 | + image="localhost/image:${VARIANT}" |
| 197 | + podman build --target "${VARIANT}" --tag "${image}" \ |
| 198 | + --label "org.opencontainers.image.version=${VERSION}" \ |
| 199 | + --label "org.opencontainers.image.revision=${CI_COMMIT_SHA}" \ |
| 200 | + --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ |
| 201 | + . |
| 202 | + - | |
| 203 | + set -euo pipefail |
| 204 | + image="localhost/image:${VARIANT}" |
| 205 | + podman run --rm "${image}" --version | grep -F " ${VERSION}" |
| 206 | + podman run --rm "${image}" devices check --help > /dev/null |
| 207 | + test "$(podman run --rm --entrypoint id "${image}" -u)" != 0 |
| 208 | + if [ "${VARIANT}" = az ]; then |
| 209 | + podman run --rm --entrypoint az "${image}" version --output none |
| 210 | + # Signed out, the tool must reach az and report it, not crash. |
| 211 | + if output="$(podman run --rm "${image}" az whoami 2>&1)"; then |
| 212 | + echo "az whoami succeeded without a sign-in" |
| 213 | + exit 1 |
| 214 | + fi |
| 215 | + grep -F "not signed in" <<< "${output}" |
| 216 | + fi |
| 217 | + - | |
| 218 | + set -euo pipefail |
| 219 | + archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" |
| 220 | + curl -fsSL -o "/tmp/${archive}" \ |
| 221 | + "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${archive}" |
| 222 | + echo "${TRIVY_SHA256} /tmp/${archive}" | sha256sum -c - |
| 223 | + tar -xzf "/tmp/${archive}" -C /tmp trivy |
| 224 | + podman save --quiet --output /tmp/image.tar "localhost/image:${VARIANT}" |
| 225 | + scan=(/tmp/trivy image --input /tmp/image.tar --cache-dir /tmp/trivy-cache --scanners vuln --quiet) |
| 226 | + # Kept with the job: every finding, unfixed included, and the SBOM. |
| 227 | + "${scan[@]}" --format table --output "trivy-${VARIANT}.txt" |
| 228 | + "${scan[@]}" --format cyclonedx --output "sbom-${VARIANT}.cdx.json" |
| 229 | + "${scan[@]}" --format table --severity HIGH,CRITICAL --ignore-unfixed |
| 230 | + # The gate: a fixed finding at this severity fails the job. |
| 231 | + "${scan[@]}" --format table --severity "${GATE}" --ignore-unfixed --exit-code 1 > /dev/null |
| 232 | + - | |
| 233 | + set -euo pipefail |
| 234 | + if [[ ! "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then |
| 235 | + echo "not a release tag: nothing to publish" |
| 236 | + exit 0 |
| 237 | + fi |
| 238 | + podman login --username "${CI_REGISTRY_USER}" --password-stdin "${CI_REGISTRY}" <<< "${CI_REGISTRY_PASSWORD}" |
| 239 | + while read -r tag; do |
| 240 | + podman push --quiet --digestfile /tmp/digest "localhost/image:${VARIANT}" \ |
| 241 | + "docker://${CI_REGISTRY_IMAGE}:${tag}" |
| 242 | + echo "pushed ${CI_REGISTRY_IMAGE}:${tag} ($(cat /tmp/digest))" |
| 243 | + done < "image-tags/${VARIANT}.txt" |
| 244 | + artifacts: |
| 245 | + when: always |
| 246 | + paths: [trivy-$VARIANT.txt, sbom-$VARIANT.cdx.json] |
| 247 | + expire_in: 30 days |
| 248 | + |
| 249 | +# Publish (release tags only) --------------------------------------------------------- |
| 250 | + |
| 251 | +# After the images, so the package never exists without them. A file already there (a |
| 252 | +# retried pipeline) is skipped, not an error. |
| 253 | +package: |
| 254 | + stage: publish |
| 255 | + needs: [build, container] |
| 256 | + interruptible: false |
| 257 | + rules: |
| 258 | + - *release-tag |
| 259 | + variables: |
| 260 | + UV_PUBLISH_URL: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi |
| 261 | + UV_PUBLISH_CHECK_URL: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi/simple |
| 262 | + UV_PUBLISH_USERNAME: gitlab-ci-token |
| 263 | + script: |
| 264 | + - UV_PUBLISH_PASSWORD="${CI_JOB_TOKEN}" uv publish dist/* |
| 265 | + |
| 266 | +# The GitLab release, last: how to install from this project's registries, and the files' |
| 267 | +# checksums. A release that is there already (a retried pipeline) is left as it is. |
| 268 | +release: |
| 269 | + stage: publish |
| 270 | + needs: [build, package] |
| 271 | + interruptible: false |
| 272 | + rules: |
| 273 | + - *release-tag |
| 274 | + script: |
| 275 | + - | |
| 276 | + python3 - <<'PY' |
| 277 | + import hashlib |
| 278 | + import json |
| 279 | + import os |
| 280 | + import pathlib |
| 281 | + import re |
| 282 | + import urllib.error |
| 283 | + import urllib.request |
| 284 | +
|
| 285 | + env = os.environ |
| 286 | + tag, version = env["CI_COMMIT_TAG"], env["VERSION"] |
| 287 | + files = sorted(pathlib.Path("dist").iterdir()) |
| 288 | + sums = "\n".join(f"{hashlib.sha256(f.read_bytes()).hexdigest()} {f.name}" for f in files) |
| 289 | + index = f"{env['CI_API_V4_URL']}/projects/{env['CI_PROJECT_ID']}/packages/pypi/simple" |
| 290 | + image = env["CI_REGISTRY_IMAGE"] |
| 291 | + final = re.fullmatch(r"\d+\.\d+\.\d+", version) is not None |
| 292 | + notes = [ |
| 293 | + "" if final else "A pre-release: installed only by asking for this version.\n", |
| 294 | + f"What changed: [CHANGELOG.md]({env['CI_PROJECT_URL']}/-/blob/{tag}/CHANGELOG.md).\n", |
| 295 | + "```bash", |
| 296 | + f"uv tool install --index {index} libre-devops-helpers=={version}", |
| 297 | + f"podman pull {image}:{version} # with the Azure CLI", |
| 298 | + f"podman pull {image}:{version}-slim # the tool alone", |
| 299 | + "```\n", |
| 300 | + "SHA256SUMS:\n", |
| 301 | + "```text", |
| 302 | + sums, |
| 303 | + "```", |
| 304 | + ] |
| 305 | + body = { |
| 306 | + "tag_name": tag, |
| 307 | + "name": tag, |
| 308 | + "description": "\n".join(notes).lstrip(), |
| 309 | + "assets": { |
| 310 | + "links": [ |
| 311 | + {"name": "Package registry", "url": f"{env['CI_PROJECT_URL']}/-/packages", "link_type": "package"}, |
| 312 | + {"name": "Container registry", "url": f"{env['CI_PROJECT_URL']}/container_registry", "link_type": "image"}, |
| 313 | + ] |
| 314 | + }, |
| 315 | + } |
| 316 | + request = urllib.request.Request( |
| 317 | + f"{env['CI_API_V4_URL']}/projects/{env['CI_PROJECT_ID']}/releases", |
| 318 | + data=json.dumps(body).encode(), |
| 319 | + headers={"JOB-TOKEN": env["CI_JOB_TOKEN"], "Content-Type": "application/json"}, |
| 320 | + method="POST", |
| 321 | + ) |
| 322 | + try: |
| 323 | + with urllib.request.urlopen(request, timeout=30) as response: |
| 324 | + print(f"created the release {tag}: {json.load(response)['_links']['self']}") |
| 325 | + except urllib.error.HTTPError as error: |
| 326 | + if error.code != 409: |
| 327 | + raise SystemExit(f"creating the release failed: HTTP {error.code} {error.read()[:500]!r}") |
| 328 | + print(f"the release {tag} is there already") |
| 329 | + PY |
0 commit comments