benchmark.yml

name: Benchmark

on:
  workflow_call:
    inputs:
      mode:
        type: string
        default: codspeed
        description: "`codspeed` (CodSpeed upload) or `compare` (pytest-benchmark, base vs head)."
      benchmark-path:
        type: string
        default: benchmarks
        description: Benchmark directory, relative to working-directory.
      extras:
        type: string
        default: dev,benchmarks
        description: Comma-separated extras to install with the wheel.
      python-version:
        type: string
        default: "3.13"
        description: Python version of the benchmark env.
      pixi-add:
        type: string
        default: ""
        description: Space-separated conda-forge specs to add to the env.
      cooldown:
        type: string
        default: 7d
        description: Ignore conda and PyPI releases newer than this age, e.g. 7d. Empty disables. uv-locked installs ignore it.
      pytest-args:
        type: string
        default: ""
        description: Extra pytest arguments.
      codspeed-modes:
        type: string
        default: '["simulation","memory"]'
        description: JSON list of CodSpeed modes for ubuntu-latest (codspeed mode only).
      walltime-label:
        type: string
        default: trigger:benchmark
        description: PR label that starts the walltime run on codspeed-macro. Non-PR events always run it (codspeed mode only).
      fail-threshold:
        type: string
        default: ""
        description: Fail if a benchmark mean regresses by more than this percent. Empty never fails (compare mode only).
      vars-env:
        type: string
        default: ""
        description: Space-separated repository or organization variable (`vars`) names to export to the job env.
      working-directory:
        type: string
        default: .
        description: Directory of the package, relative to the repo root.
    secrets:
      CDSAPI_KEY:
        required: false
      EARTHDATAHUB_API_KEY:
        required: false
      MOSEKLM_LICENSE_FILE:
        required: false
      GUROBI_LIC:
        required: false

permissions: {}

jobs:
  build:
    if: inputs.mode == 'codspeed'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      legs: ${{ steps.legs.outputs.legs }}
    steps:
      - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
        with:
          egress-policy: audit
      - if: github.event_name == 'pull_request_target'
        run: |
          echo "::error::pull_request_target runs fork code with secrets. Use pull_request." >&2
          exit 1
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false
      - uses: hynek/build-and-inspect-python-package@67230f7b8410ae0052b39c1efc89d89fff43ff10 # v3.0.2
        with:
          path: ${{ inputs.working-directory }}
          skip-sdist: true
          upload-name-suffix: -benchmark
      - id: legs
        env:
          MODES: ${{ inputs.codspeed-modes }}
          WALLTIME: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, inputs.walltime-label) }}
        run: |
          set -euo pipefail
          legs=$(jq -c --argjson walltime "$WALLTIME" \
            '[.[] | {mode: ., runner: "ubuntu-latest"}] + (if $walltime then [{mode: "walltime", runner: "codspeed-macro"}] else [] end)' \
            <<< "$MODES")
          echo "legs=$legs" >> "$GITHUB_OUTPUT"

  codspeed:
    needs: build
    name: codspeed (${{ matrix.mode }})
    runs-on: ${{ matrix.runner }}
    continue-on-error: true
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.build.outputs.legs) }}
    steps:
      - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
        if: matrix.runner != 'codspeed-macro'
        with:
          egress-policy: audit
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: Packages-benchmark
          path: ${{ runner.temp }}/wheels
      - uses: $/.github/actions/setup-env
        with:
          mode: library
          python-version: ${{ inputs.python-version }}
          pixi-add: ${{ inputs.pixi-add }}
          extras: ${{ inputs.extras }}
          cooldown: ${{ inputs.cooldown }}
          wheel-dir: ${{ runner.temp }}/wheels
          vars-json: ${{ toJSON(vars) }}
          vars-env: ${{ inputs.vars-env }}
          working-directory: ${{ inputs.working-directory }}
      - uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0
        env:
          BENCHMARK_PATH: ${{ inputs.benchmark-path }}
          PYTEST_ARGS: ${{ inputs.pytest-args }}
          CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}
          EARTHDATAHUB_API_KEY: ${{ secrets.EARTHDATAHUB_API_KEY }}
          MOSEKLM_LICENSE_FILE: ${{ secrets.MOSEKLM_LICENSE_FILE }}
          GUROBI_LIC: ${{ secrets.GUROBI_LIC }}
        with:
          mode: ${{ matrix.mode }}
          working-directory: ${{ inputs.working-directory }}
          run: $RUN pytest "$BENCHMARK_PATH" --codspeed $PYTEST_ARGS

  compare:
    if: inputs.mode == 'compare'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    env:
      BENCHMARK_PATH: ${{ inputs.benchmark-path }}
      PYTEST_ARGS: ${{ inputs.pytest-args }}
      EXTRAS: ${{ inputs.extras }}
      IS_PR: ${{ github.event_name == 'pull_request' }}
    steps:
      - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
        with:
          egress-policy: audit
      - if: github.event_name == 'pull_request_target'
        run: |
          echo "::error::pull_request_target runs fork code with secrets. Use pull_request." >&2
          exit 1
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          path: head
          fetch-depth: 0
          persist-credentials: false
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: github.event_name == 'pull_request'
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: base
          fetch-depth: 0
          persist-credentials: false
      - id: base
        env:
          WORKING_DIRECTORY: ${{ inputs.working-directory }}
        run: |
          set -euo pipefail
          if [ "$IS_PR" = true ] && [ -f "base/$WORKING_DIRECTORY/pyproject.toml" ]; then
            echo "present=true" >> "$GITHUB_OUTPUT"
          else
            echo "present=false" >> "$GITHUB_OUTPUT"
          fi
      - uses: hynek/build-and-inspect-python-package@67230f7b8410ae0052b39c1efc89d89fff43ff10 # v3.0.2
        id: head-build
        with:
          path: head/${{ inputs.working-directory }}
          skip-sdist: true
          upload-name-suffix: -benchmark-head
      - id: head-wheel
        env:
          DIST: ${{ steps.head-build.outputs.dist }}
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/wheels/head" "$RUNNER_TEMP/wheels/base"
          cp "$DIST"/*.whl "$RUNNER_TEMP/wheels/head/"
          rm -rf /tmp/baipp
          echo "wheel=$(ls "$RUNNER_TEMP"/wheels/head/*.whl)" >> "$GITHUB_OUTPUT"
      - uses: hynek/build-and-inspect-python-package@67230f7b8410ae0052b39c1efc89d89fff43ff10 # v3.0.2
        id: base-build
        if: steps.base.outputs.present == 'true'
        with:
          path: base/${{ inputs.working-directory }}
          skip-sdist: true
          upload-name-suffix: -benchmark-base
      - id: wheels
        env:
          BASE_DIST: ${{ steps.base-build.outputs.dist }}
          BASE_PRESENT: ${{ steps.base.outputs.present }}
        run: |
          set -euo pipefail
          first="head"
          if [ "$BASE_PRESENT" = true ]; then
            cp "$BASE_DIST"/*.whl "$RUNNER_TEMP/wheels/base/"
            first="base"
          fi
          echo "first=$RUNNER_TEMP/wheels/$first" >> "$GITHUB_OUTPUT"
      - uses: $/.github/actions/setup-env
        with:
          mode: library
          python-version: ${{ inputs.python-version }}
          pixi-add: ${{ inputs.pixi-add }}
          extras: ${{ inputs.extras }}
          cooldown: ${{ inputs.cooldown }}
          wheel-dir: ${{ steps.wheels.outputs.first }}
          vars-json: ${{ toJSON(vars) }}
          vars-env: ${{ inputs.vars-env }}
          working-directory: ${{ steps.base.outputs.present == 'true' && 'base' || 'head' }}/${{ inputs.working-directory }}
      - if: steps.base.outputs.present == 'true'
        working-directory: base/${{ inputs.working-directory }}
        env:
          CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}
          EARTHDATAHUB_API_KEY: ${{ secrets.EARTHDATAHUB_API_KEY }}
          MOSEKLM_LICENSE_FILE: ${{ secrets.MOSEKLM_LICENSE_FILE }}
          GUROBI_LIC: ${{ secrets.GUROBI_LIC }}
        run: |
          set -euo pipefail
          read -ra pytest_args <<< "$PYTEST_ARGS"
          if [ -e "$BENCHMARK_PATH" ]; then
            $RUN pytest "$BENCHMARK_PATH" --benchmark-only --benchmark-json="$RUNNER_TEMP/base.json" "${pytest_args[@]}"
          else
            echo "Base has no $BENCHMARK_PATH. Skip the base run."
          fi
      - if: steps.base.outputs.present == 'true'
        env:
          PACKAGE: ${{ steps.head-build.outputs.package_name }}
          WHEEL: ${{ steps.head-wheel.outputs.wheel }}
        run: $RUN uv pip install --reinstall-package "$PACKAGE" "${WHEEL}[$EXTRAS]"
      - working-directory: head/${{ inputs.working-directory }}
        env:
          CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}
          EARTHDATAHUB_API_KEY: ${{ secrets.EARTHDATAHUB_API_KEY }}
          MOSEKLM_LICENSE_FILE: ${{ secrets.MOSEKLM_LICENSE_FILE }}
          GUROBI_LIC: ${{ secrets.GUROBI_LIC }}
        run: |
          set -euo pipefail
          read -ra pytest_args <<< "$PYTEST_ARGS"
          $RUN pytest "$BENCHMARK_PATH" --benchmark-only --benchmark-json="$RUNNER_TEMP/head.json" "${pytest_args[@]}"
      - working-directory: head/${{ inputs.working-directory }}
        run: |
          set -euo pipefail
          report="$RUNNER_TEMP/benchmark.md"
          files=("$RUNNER_TEMP/head.json")
          title="Benchmark results"
          [ "$IS_PR" != true ] || title="Benchmark results (no base to compare)"
          if [ -f "$RUNNER_TEMP/base.json" ]; then
            files=("$RUNNER_TEMP/base.json" "$RUNNER_TEMP/head.json")
            title="Benchmark comparison: base (0001) vs head (0002)"
          fi
          {
            echo "## $title"
            echo '```'
            $RUN pytest-benchmark compare "${files[@]}" --columns=mean,stddev --group-by=name
            echo '```'
          } > "$report"
          cat "$report" >> "$GITHUB_STEP_SUMMARY"
      - uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        with:
          header: benchmark
          path: ${{ runner.temp }}/benchmark.md
      - if: github.event_name == 'pull_request' && inputs.fail-threshold != ''
        env:
          FAIL_THRESHOLD: ${{ inputs.fail-threshold }}
        run: |
          set -euo pipefail
          [ -f "$RUNNER_TEMP/base.json" ] || exit 0
          python3 - "$RUNNER_TEMP/base.json" "$RUNNER_TEMP/head.json" <<'PY'
          import json, os, sys
          load = lambda p: {b["fullname"]: b["stats"]["mean"] for b in json.load(open(p))["benchmarks"]}
          base, head = load(sys.argv[1]), load(sys.argv[2])
          limit = float(os.environ["FAIL_THRESHOLD"])
          changes = {k: 100 * (head[k] / base[k] - 1) for k in base.keys() & head.keys()}
          bad = {k: v for k, v in changes.items() if v > limit}
          for k, v in sorted(bad.items()):
              print(f"::error::{k}: mean +{v:.1f}% (limit {limit}%)")
          sys.exit(bool(bad))
          PY