From 7eaa2edeaede0ab6d9c66869f8a9b910aaf8a640 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:34:41 +0200 Subject: [PATCH 1/9] chore(probe): measure the hydra-gates guard in all three producer states Temporary harness. Reproduces quality.yml's hydra-gates topology exactly (needs: [playwright] + if: inputs.enable-hydra-gates && !cancelled()) and runs it three times in one run: producer succeeded, failed, skipped. Also measures the candidate always()-based guard side by side. --- .../workflows/_probe-gate-guard-caller.yml | 30 +++++++++ .github/workflows/_probe-gate-guard.yml | 65 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 .github/workflows/_probe-gate-guard-caller.yml create mode 100644 .github/workflows/_probe-gate-guard.yml diff --git a/.github/workflows/_probe-gate-guard-caller.yml b/.github/workflows/_probe-gate-guard-caller.yml new file mode 100644 index 0000000..df7e7ff --- /dev/null +++ b/.github/workflows/_probe-gate-guard-caller.yml @@ -0,0 +1,30 @@ +# TEMPORARY MEASUREMENT HARNESS — see _probe-gate-guard.yml. Deleted after the +# measurement is recorded. +name: probe-gate-guard-caller + +on: + push: + branches: [fix/hydra-gates-needs-skip-deletion] + workflow_dispatch: + +jobs: + producer-succeeded: + uses: ./.github/workflows/_probe-gate-guard.yml + with: + enable-playwright: true + playwright-fails: false + enable-hydra-gates: true + + producer-failed: + uses: ./.github/workflows/_probe-gate-guard.yml + with: + enable-playwright: true + playwright-fails: true + enable-hydra-gates: true + + producer-skipped: + uses: ./.github/workflows/_probe-gate-guard.yml + with: + enable-playwright: false + playwright-fails: false + enable-hydra-gates: true diff --git a/.github/workflows/_probe-gate-guard.yml b/.github/workflows/_probe-gate-guard.yml new file mode 100644 index 0000000..0ec26f5 --- /dev/null +++ b/.github/workflows/_probe-gate-guard.yml @@ -0,0 +1,65 @@ +# TEMPORARY MEASUREMENT HARNESS — not part of the quality pipeline. +# +# Reproduces the exact `needs:` / `if:` topology of the `hydra-gates` job in +# quality.yml so the guard can be measured in all three producer states +# (playwright succeeded / failed / skipped) in ONE run. Deleted after the +# measurement is recorded. +name: probe-gate-guard + +on: + workflow_call: + inputs: + enable-playwright: + type: boolean + required: true + playwright-fails: + type: boolean + required: true + enable-hydra-gates: + type: boolean + required: true + +jobs: + # Stands in for quality.yml's `playwright` job, which carries the same + # `inputs. && !cancelled()` shape. + playwright: + if: ${{ inputs.enable-playwright && !cancelled() }} + runs-on: ubuntu-latest + name: "E2E Tests (Playwright)" + steps: + - name: Be the producer + run: | + if [ "${{ inputs.playwright-fails }}" = "true" ]; then + echo "producer FAILS" + exit 1 + fi + echo "producer SUCCEEDS" + + # The guard as shipped by .github#148. + gates-current: + if: ${{ inputs.enable-hydra-gates && !cancelled() }} + needs: [playwright] + runs-on: ubuntu-latest + name: "Hydra Gates (current guard)" + steps: + - run: echo "GATES RAN. needs.playwright.result=${{ needs.playwright.result }}" + + # The candidate replacement. + gates-always: + if: ${{ always() && inputs.enable-hydra-gates }} + needs: [playwright] + runs-on: ubuntu-latest + name: "Hydra Gates (always guard)" + steps: + - run: echo "GATES RAN. needs.playwright.result=${{ needs.playwright.result }}" + + report: + needs: [playwright, gates-current, gates-always] + if: always() + runs-on: ubuntu-latest + name: "Report" + steps: + - run: | + echo "playwright=${{ needs.playwright.result }}" + echo "gates-current=${{ needs.gates-current.result }}" + echo "gates-always=${{ needs.gates-always.result }}" From b856418f7cc1b663b833d32d0bcc9e87c86b3de0 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:41:39 +0200 Subject: [PATCH 2/9] =?UTF-8?q?chore(probe):=20A/B=20arm=20'before'=20?= =?UTF-8?q?=E2=80=94=20quality.yml@main=20with=20axe=20on=20and=20its=20pr?= =?UTF-8?q?oducer=20off?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/_probe-axe-failclosed.yml | 48 +++++++++++ .../workflows/_probe-gate-guard-caller.yml | 4 +- .github/workflows/quality.yml | 80 +++++++++++++++---- 3 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/_probe-axe-failclosed.yml diff --git a/.github/workflows/_probe-axe-failclosed.yml b/.github/workflows/_probe-axe-failclosed.yml new file mode 100644 index 0000000..0002d2e --- /dev/null +++ b/.github/workflows/_probe-axe-failclosed.yml @@ -0,0 +1,48 @@ +# TEMPORARY MEASUREMENT HARNESS — deleted once the measurement is recorded. +# +# A/B on the REAL quality.yml, not a copy of it. Both arms are the same caller +# with the same inputs; the arms differ ONLY in which version of quality.yml +# they call: +# +# ARM = before -> ConductionNL/.github/.github/workflows/quality.yml@main +# ARM = after -> ./.github/workflows/quality.yml (this branch) +# +# Inputs put the pipeline in the exact hole under test: the gates are ON, axe +# enforcement is ON, and the only producer of tests/axe/report.json — the +# Playwright job — is OFF, so no report can possibly reach the gates job. +# +# Expected: `Hydra Gates` SUCCEEDS on main (gate-33 silently skipped, no +# accessibility measured, plain green tick) and FAILS on this branch with a +# named reason. Run one arm per push: both arms in one run would collide on the +# `result-hydra-gates` artifact name and the collision, not the change, would +# decide the verdict. +name: probe-axe-failclosed + +on: + push: + branches: [fix/hydra-gates-needs-skip-deletion] + +permissions: + contents: write + actions: write + issues: write + pull-requests: write + +jobs: + # ARM = before + arm: + uses: ConductionNL/.github/.github/workflows/quality.yml@main + with: + app-name: probe + enable-hydra-gates: true + enable-axe: true + enable-playwright: false + enable-phpunit: false + enable-newman: false + enable-frontend: false + enable-eslint: false + enable-psalm: false + enable-phpstan: false + enable-phpmetrics: false + enable-sbom: false + enable-coverage-guard: false diff --git a/.github/workflows/_probe-gate-guard-caller.yml b/.github/workflows/_probe-gate-guard-caller.yml index df7e7ff..2760a7f 100644 --- a/.github/workflows/_probe-gate-guard-caller.yml +++ b/.github/workflows/_probe-gate-guard-caller.yml @@ -3,8 +3,8 @@ name: probe-gate-guard-caller on: - push: - branches: [fix/hydra-gates-needs-skip-deletion] + # `push` was removed after run 30909727531 recorded the measurement, so later + # pushes to this branch do not re-run a harness that has already answered. workflow_dispatch: jobs: diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 11cd72d..640f421 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -163,7 +163,7 @@ on: # almost certainly be red, and some of that red will not be the app's # to fix. Default-on would therefore have blocked the fleet on core # Nextcloud defects on day one. - description: "Run axe-core (@axe-core/playwright) against the app's routes inside the Playwright job and publish tests/axe/report.json, which is the file hydra-gates gate-33 consumes. OPT-IN, default false. Gate-33 fails on `serious`/`critical` violations only; without this the report does not exist and gate-33 reports SKIPPED, which is what it has done in every repo in the fleet since it was written. Requires enable-playwright (the report is produced by that job, against its Nextcloud instance) and is only consumed when enable-hydra-gates is also true. Measured against a vanilla Nextcloud 34 with no app installed, core's own pages already carry serious/critical violations — expect the first run in a repo to be red, which is exactly why this is off by default." + description: "Run axe-core (@axe-core/playwright) against the app's routes inside the Playwright job and publish tests/axe/report.json, which is the file hydra-gates gate-33 consumes. OPT-IN, default false. Gate-33 fails on `serious`/`critical` violations only; without this the report does not exist and gate-33 reports SKIPPED, which is what it has done in every repo in the fleet since it was written. Requires enable-playwright (the report is produced by that job, against its Nextcloud instance) and is only consumed when enable-hydra-gates is also true. SETTING THIS TRUE IS AN OPT-IN TO ENFORCEMENT, NOT TO A BEST EFFORT: if no usable axe report reaches the hydra-gates job — because Playwright was skipped or failed, or because the report was rejected by the provenance check — that job now FAILS with a named reason instead of going green with gate-33 silently SKIPPED. Leave it false, deliberately and visibly, for any repo not ready to enforce it. Measured against a vanilla Nextcloud 34 with no app installed, core's own pages already carry serious/critical violations — expect the first run in a repo to be red, which is exactly why this is off by default." required: false type: boolean default: false @@ -3210,8 +3210,30 @@ jobs: # stop running on a majority of the fleet, silently, as a side effect of # wiring up ONE gate's input. `!cancelled()` suppresses that implicit # `success()`, so a skipped or failed Playwright job still lets the gates - # run and reach their own verdict. The axe report is then simply absent and - # gate-33 reports SKIPPED, exactly as it does today. + # run and reach their own verdict. + # + # MEASURED, not assumed — ConductionNL/.github run 30909727531, a harness + # that reproduced this exact `needs:`/`if:` pair three times in one run and + # drove the producer into each of its three states. This guard, and the + # `always() && inputs.enable-hydra-gates` alternative, behaved identically + # and correctly in all three: + # + # producer succeeded -> gates RAN (needs.playwright.result=success) + # producer FAILED -> gates RAN (needs.playwright.result=failure) + # producer SKIPPED -> gates RAN (needs.playwright.result=skipped) + # + # The job never vanished. So `!cancelled()` is sufficient and there is no + # reason to reach for `always()`, which would additionally run the gates + # during a cancellation and produce a verdict nobody asked for. Do not + # "simplify" this back to a bare `inputs.enable-hydra-gates`: that restores + # the implicit `success()` and deletes the job with no red, no skip and no + # trace whenever Playwright does not succeed. + # + # What the measurement also settled: keeping the job alive is the RIGHT + # behaviour for the other 60 gates and precisely what makes gate-33 + # dangerous, because a live job with an absent artifact used to go green + # having measured no accessibility at all. That hole is closed at the + # `enable-axe` step below, not here. if: ${{ inputs.enable-hydra-gates && !cancelled() }} runs-on: ubuntu-latest name: "Hydra Gates" @@ -3276,11 +3298,11 @@ jobs: # ── gate-33's subject, handed over from the Playwright job ───────────── # - # continue-on-error because the artifact legitimately does not exist when - # the Playwright job was skipped or its axe step refused to write a - # report. That case is not an error here: gate-33 then reports SKIPPED - # with its reason, which is the honest outcome and the one this repo has - # had all along. + # continue-on-error so that a missing artifact does not abort the job + # here with download-artifact's own generic message. The absence is + # handled one step down, where it can be named — and, since `enable-axe` + # is an explicit opt-in to enforcement, where it FAILS. It is not + # tolerated: continue-on-error moves the verdict, it does not soften it. - name: Download the axe-core report id: axe-download if: ${{ inputs.enable-axe }} @@ -3298,16 +3320,34 @@ jobs: # "never measured" into "measured clean". # # So the file is checked against the provenance the runner writes, and - # DELETED if it does not hold up. Deleting it restores the loud skip; it - # never invents a pass and it never invents a failure. - - name: Validate the axe report before gate-33 reads it + # DELETED if it does not hold up — gate-33 can then never read a rejected + # file as a clean run. + # + # Deletion alone is not enough, and this is the second half of the same + # defect. `enable-axe: true` is an explicit request for accessibility + # ENFORCEMENT, not for a best-effort attempt. Before this step failed + # closed, the two ways of having no measurement — the artifact never + # arrived (Playwright skipped, or crashed before its axe step), and the + # artifact arrived but was rejected here — both ended in gate-33 + # reporting SKIPPED and the gates job reporting a plain green tick. A + # `::notice` buried in a three-thousand-line log is not a verdict: the + # absence of the check looked exactly like the check succeeding, which is + # the shape this whole file exists to eliminate. + # + # So with `enable-axe` set, no usable report is a FAILURE with a named + # reason and the producer's resolved state quoted in it. The way to not + # enforce axe is `enable-axe: false` — one line in the caller, visible in + # a diff and attributable to a person — not a silent green in a log. + - name: The axe report must exist and hold up if: ${{ inputs.enable-axe }} + env: + PLAYWRIGHT_RESULT: ${{ needs.playwright.result }} run: | set -uo pipefail REPORT="${GITHUB_WORKSPACE}/app/tests/axe/report.json" if [ ! -f "${REPORT}" ]; then - echo "::notice::No axe report was downloaded. gate-33 will report SKIPPED with its reason — accessibility runtime checking is UNVERIFIED for this run." - exit 0 + echo "::error::enable-axe is set, but NO axe report reached this job — gate-33 measured nothing and this run carries no accessibility verdict at all. The Playwright job, which is the only producer of tests/axe/report.json, resolved '${PLAYWRIGHT_RESULT}'. Fix that job (enable-axe also requires enable-playwright), or set enable-axe: false to state deliberately that this repo does not enforce axe yet." + exit 1 fi echo "Downloaded $(wc -c < "${REPORT}") byte(s) into ${REPORT}." python3 - "${REPORT}" <<'PYVALIDATE' && VALID=0 || VALID=$? @@ -3348,7 +3388,9 @@ jobs: PYVALIDATE if [ "${VALID}" -ne 0 ]; then rm -f "${REPORT}" - echo "Removed ${REPORT}. gate-33 will now report SKIPPED rather than PASS." + echo "Removed ${REPORT} so gate-33 cannot read it as a clean accessibility run." + echo "::error::enable-axe is set and the axe report was REJECTED for the reason above, so this run has no accessibility verdict. A rejected report is the same amount of evidence as no report: none. Fix the producer, or set enable-axe: false." + exit 1 fi - name: Resolve the diff base @@ -3556,6 +3598,16 @@ jobs: echo "| PHPUnit | | | | | $(icon '${{ needs.phpunit.result }}') |" echo "| Newman | | | | | $(icon '${{ needs.newman.result }}') |" echo "| Playwright | | | | | $(icon '${{ needs.playwright.result }}') |" + # The Hydra gates were in this job's `needs:` and in its failure + # gate below, so a red gates job did red the Quality Report — but + # the report itself never mentioned them, in any state. The one + # artefact people actually read (the PR comment, the PDF) was + # silent about sixty gates: read on its own it was indistinguishable + # from a repo that has no gates, whether they passed, failed or + # never ran. `needs..result` rather than the result artifact, + # so a job that was skipped or deleted still renders as ⏭️ instead + # of vanishing from the table. + echo "| Hydra gates | | | | | $(icon '${{ needs.hydra-gates.result }}') |" echo "" } > "$REPORT" From 63e8d9f093f051545fb587c666e9f3f68384fbf8 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:43:05 +0200 Subject: [PATCH 3/9] =?UTF-8?q?chore(probe):=20A/B=20arm=20'after'=20?= =?UTF-8?q?=E2=80=94=20same=20inputs=20against=20this=20branch's=20quality?= =?UTF-8?q?.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/_probe-axe-failclosed.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_probe-axe-failclosed.yml b/.github/workflows/_probe-axe-failclosed.yml index 0002d2e..8ff6eee 100644 --- a/.github/workflows/_probe-axe-failclosed.yml +++ b/.github/workflows/_probe-axe-failclosed.yml @@ -29,9 +29,10 @@ permissions: pull-requests: write jobs: - # ARM = before + # ARM = after. ARM = before was run 30910234160 (Hydra Gates: SUCCESS with + # gate-33 SKIPPED and zero routes measured). arm: - uses: ConductionNL/.github/.github/workflows/quality.yml@main + uses: ./.github/workflows/quality.yml with: app-name: probe enable-hydra-gates: true From cad826d18e6de548255de8137aaa7c7c58bd8df7 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:50:18 +0200 Subject: [PATCH 4/9] fix(quality): raise the axe verdict at the end so it stops suppressing the other 60 gates --- .github/workflows/quality.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 640f421..fd2732a 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -3338,7 +3338,17 @@ jobs: # reason and the producer's resolved state quoted in it. The way to not # enforce axe is `enable-axe: false` — one line in the caller, visible in # a diff and attributable to a person — not a silent green in a log. + # + # The failure is RECORDED here and RAISED at the end of the job, not + # thrown from this step. Measured first as a plain `exit 1` (run + # 30910345294): it aborted the job before `Run the Hydra gates`, so a + # missing axe artifact took the other sixty gates down with it and the PR + # lost all mechanical feedback until someone fixed an unrelated Playwright + # job. A gate that suppresses sixty other gates to report itself is its + # own kind of outage. So this step always exits 0, the gates run and print + # their full report, and the job is failed afterwards. - name: The axe report must exist and hold up + id: axe-check if: ${{ inputs.enable-axe }} env: PLAYWRIGHT_RESULT: ${{ needs.playwright.result }} @@ -3346,8 +3356,9 @@ jobs: set -uo pipefail REPORT="${GITHUB_WORKSPACE}/app/tests/axe/report.json" if [ ! -f "${REPORT}" ]; then - echo "::error::enable-axe is set, but NO axe report reached this job — gate-33 measured nothing and this run carries no accessibility verdict at all. The Playwright job, which is the only producer of tests/axe/report.json, resolved '${PLAYWRIGHT_RESULT}'. Fix that job (enable-axe also requires enable-playwright), or set enable-axe: false to state deliberately that this repo does not enforce axe yet." - exit 1 + echo "unmeasured=absent" >> "$GITHUB_OUTPUT" + echo "::error::enable-axe is set, but NO axe report reached this job — gate-33 will measure nothing and this run carries no accessibility verdict at all. The Playwright job, which is the only producer of tests/axe/report.json, resolved '${PLAYWRIGHT_RESULT}'. Fix that job (enable-axe also requires enable-playwright), or set enable-axe: false to state deliberately that this repo does not enforce axe yet. The remaining gates still run below; this job will fail at the end." + exit 0 fi echo "Downloaded $(wc -c < "${REPORT}") byte(s) into ${REPORT}." python3 - "${REPORT}" <<'PYVALIDATE' && VALID=0 || VALID=$? @@ -3388,9 +3399,9 @@ jobs: PYVALIDATE if [ "${VALID}" -ne 0 ]; then rm -f "${REPORT}" + echo "unmeasured=rejected" >> "$GITHUB_OUTPUT" echo "Removed ${REPORT} so gate-33 cannot read it as a clean accessibility run." - echo "::error::enable-axe is set and the axe report was REJECTED for the reason above, so this run has no accessibility verdict. A rejected report is the same amount of evidence as no report: none. Fix the producer, or set enable-axe: false." - exit 1 + echo "::error::enable-axe is set and the axe report was REJECTED for the reason above, so this run has no accessibility verdict. A rejected report is the same amount of evidence as no report: none. Fix the producer, or set enable-axe: false. The remaining gates still run below; this job will fail at the end." fi - name: Resolve the diff base @@ -3447,6 +3458,18 @@ jobs: echo "::error::${RC} Hydra gate(s) failed. The exit code is the failure COUNT — see the named [gate-N] FAIL lines above." exit 1 + # The verdict for `enable-axe`, raised here rather than at the check + # itself so the gates above still produce their full report first (see + # the reasoning on the axe-check step). `always()` so it is reached even + # when the gates step already failed — a run can be both "gates failed" + # and "accessibility never measured", and collapsing those loses one. + # It only ever turns a pass into a failure, never the reverse. + - name: Fail when axe was requested but never measured + if: ${{ always() && inputs.enable-axe && steps.axe-check.outputs.unmeasured != '' }} + run: | + echo "::error::enable-axe is set and this run produced NO accessibility measurement (reason: ${{ steps.axe-check.outputs.unmeasured }}). gate-33 reporting SKIPPED is the absence of the check, not the check passing — see the named error earlier in this job. Failing the gates job so the run cannot be read as accessible." + exit 1 + - name: Record result if: always() run: | From c230c301b0f8f33bb9907badf1bc4e72af7b8d13 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:52:19 +0200 Subject: [PATCH 5/9] =?UTF-8?q?chore(probe):=20no-regression=20arm=20?= =?UTF-8?q?=E2=80=94=20enable-axe=20false,=20the=20fleet=20default=20(open?= =?UTF-8?q?build's=20config)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/_probe-axe-failclosed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_probe-axe-failclosed.yml b/.github/workflows/_probe-axe-failclosed.yml index 8ff6eee..ffd7618 100644 --- a/.github/workflows/_probe-axe-failclosed.yml +++ b/.github/workflows/_probe-axe-failclosed.yml @@ -36,7 +36,7 @@ jobs: with: app-name: probe enable-hydra-gates: true - enable-axe: true + enable-axe: false enable-playwright: false enable-phpunit: false enable-newman: false From 120b5c7bc4149bcd8715e9099f0fdb92c01f3bc3 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:53:44 +0200 Subject: [PATCH 6/9] chore(probe): remove the measurement harnesses; the run IDs are recorded in quality.yml --- .github/workflows/_probe-axe-failclosed.yml | 49 -------------- .../workflows/_probe-gate-guard-caller.yml | 30 --------- .github/workflows/_probe-gate-guard.yml | 65 ------------------- 3 files changed, 144 deletions(-) delete mode 100644 .github/workflows/_probe-axe-failclosed.yml delete mode 100644 .github/workflows/_probe-gate-guard-caller.yml delete mode 100644 .github/workflows/_probe-gate-guard.yml diff --git a/.github/workflows/_probe-axe-failclosed.yml b/.github/workflows/_probe-axe-failclosed.yml deleted file mode 100644 index ffd7618..0000000 --- a/.github/workflows/_probe-axe-failclosed.yml +++ /dev/null @@ -1,49 +0,0 @@ -# TEMPORARY MEASUREMENT HARNESS — deleted once the measurement is recorded. -# -# A/B on the REAL quality.yml, not a copy of it. Both arms are the same caller -# with the same inputs; the arms differ ONLY in which version of quality.yml -# they call: -# -# ARM = before -> ConductionNL/.github/.github/workflows/quality.yml@main -# ARM = after -> ./.github/workflows/quality.yml (this branch) -# -# Inputs put the pipeline in the exact hole under test: the gates are ON, axe -# enforcement is ON, and the only producer of tests/axe/report.json — the -# Playwright job — is OFF, so no report can possibly reach the gates job. -# -# Expected: `Hydra Gates` SUCCEEDS on main (gate-33 silently skipped, no -# accessibility measured, plain green tick) and FAILS on this branch with a -# named reason. Run one arm per push: both arms in one run would collide on the -# `result-hydra-gates` artifact name and the collision, not the change, would -# decide the verdict. -name: probe-axe-failclosed - -on: - push: - branches: [fix/hydra-gates-needs-skip-deletion] - -permissions: - contents: write - actions: write - issues: write - pull-requests: write - -jobs: - # ARM = after. ARM = before was run 30910234160 (Hydra Gates: SUCCESS with - # gate-33 SKIPPED and zero routes measured). - arm: - uses: ./.github/workflows/quality.yml - with: - app-name: probe - enable-hydra-gates: true - enable-axe: false - enable-playwright: false - enable-phpunit: false - enable-newman: false - enable-frontend: false - enable-eslint: false - enable-psalm: false - enable-phpstan: false - enable-phpmetrics: false - enable-sbom: false - enable-coverage-guard: false diff --git a/.github/workflows/_probe-gate-guard-caller.yml b/.github/workflows/_probe-gate-guard-caller.yml deleted file mode 100644 index 2760a7f..0000000 --- a/.github/workflows/_probe-gate-guard-caller.yml +++ /dev/null @@ -1,30 +0,0 @@ -# TEMPORARY MEASUREMENT HARNESS — see _probe-gate-guard.yml. Deleted after the -# measurement is recorded. -name: probe-gate-guard-caller - -on: - # `push` was removed after run 30909727531 recorded the measurement, so later - # pushes to this branch do not re-run a harness that has already answered. - workflow_dispatch: - -jobs: - producer-succeeded: - uses: ./.github/workflows/_probe-gate-guard.yml - with: - enable-playwright: true - playwright-fails: false - enable-hydra-gates: true - - producer-failed: - uses: ./.github/workflows/_probe-gate-guard.yml - with: - enable-playwright: true - playwright-fails: true - enable-hydra-gates: true - - producer-skipped: - uses: ./.github/workflows/_probe-gate-guard.yml - with: - enable-playwright: false - playwright-fails: false - enable-hydra-gates: true diff --git a/.github/workflows/_probe-gate-guard.yml b/.github/workflows/_probe-gate-guard.yml deleted file mode 100644 index 0ec26f5..0000000 --- a/.github/workflows/_probe-gate-guard.yml +++ /dev/null @@ -1,65 +0,0 @@ -# TEMPORARY MEASUREMENT HARNESS — not part of the quality pipeline. -# -# Reproduces the exact `needs:` / `if:` topology of the `hydra-gates` job in -# quality.yml so the guard can be measured in all three producer states -# (playwright succeeded / failed / skipped) in ONE run. Deleted after the -# measurement is recorded. -name: probe-gate-guard - -on: - workflow_call: - inputs: - enable-playwright: - type: boolean - required: true - playwright-fails: - type: boolean - required: true - enable-hydra-gates: - type: boolean - required: true - -jobs: - # Stands in for quality.yml's `playwright` job, which carries the same - # `inputs. && !cancelled()` shape. - playwright: - if: ${{ inputs.enable-playwright && !cancelled() }} - runs-on: ubuntu-latest - name: "E2E Tests (Playwright)" - steps: - - name: Be the producer - run: | - if [ "${{ inputs.playwright-fails }}" = "true" ]; then - echo "producer FAILS" - exit 1 - fi - echo "producer SUCCEEDS" - - # The guard as shipped by .github#148. - gates-current: - if: ${{ inputs.enable-hydra-gates && !cancelled() }} - needs: [playwright] - runs-on: ubuntu-latest - name: "Hydra Gates (current guard)" - steps: - - run: echo "GATES RAN. needs.playwright.result=${{ needs.playwright.result }}" - - # The candidate replacement. - gates-always: - if: ${{ always() && inputs.enable-hydra-gates }} - needs: [playwright] - runs-on: ubuntu-latest - name: "Hydra Gates (always guard)" - steps: - - run: echo "GATES RAN. needs.playwright.result=${{ needs.playwright.result }}" - - report: - needs: [playwright, gates-current, gates-always] - if: always() - runs-on: ubuntu-latest - name: "Report" - steps: - - run: | - echo "playwright=${{ needs.playwright.result }}" - echo "gates-current=${{ needs.gates-current.result }}" - echo "gates-always=${{ needs.gates-always.result }}" From e9f27ea02662b2e15d525de3a849a515b3f366fd Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:53:51 +0200 Subject: [PATCH 7/9] docs(quality): record the A/B run IDs and the no-regression control in the job --- .github/workflows/quality.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index fd2732a..e424dce 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -3339,6 +3339,21 @@ jobs: # enforce axe is `enable-axe: false` — one line in the caller, visible in # a diff and attributable to a person — not a silent green in a log. # + # A/B on this file, same caller and same inputs (gates on, axe on, + # Playwright — the only producer — off), the arms differing only in which + # quality.yml they called: + # + # before, quality.yml@main : run 30910234160, Hydra Gates SUCCESS + # "##[notice]No axe report was downloaded" + # "[gate-33] axe-core: SKIPPED" + # after, this branch : run 30910908474, Hydra Gates FAILURE + # gates still ran and reported in full + # + # And the control that bounds the blast radius: the same inputs with + # `enable-axe: false` — the fleet default and openbuild's live config, the + # only repo with the gates enabled — run 30911069404, Hydra Gates SUCCESS + # with both axe steps SKIPPED. Nothing that is green today turns red. + # # The failure is RECORDED here and RAISED at the end of the job, not # thrown from this step. Measured first as a plain `exit 1` (run # 30910345294): it aborted the job before `Run the Hydra gates`, so a From 91acd513c1540b13e842eb66f9de4a2130b80a67 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 14:57:51 +0200 Subject: [PATCH 8/9] =?UTF-8?q?chore(probe):=20third=20producer=20state=20?= =?UTF-8?q?=E2=80=94=20Playwright=20ON=20and=20failing,=20so=20the=20produ?= =?UTF-8?q?cer=20fails=20rather=20than=20skips?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/_probe-axe-producer-failed.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/_probe-axe-producer-failed.yml diff --git a/.github/workflows/_probe-axe-producer-failed.yml b/.github/workflows/_probe-axe-producer-failed.yml new file mode 100644 index 0000000..23bb480 --- /dev/null +++ b/.github/workflows/_probe-axe-producer-failed.yml @@ -0,0 +1,37 @@ +# TEMPORARY MEASUREMENT HARNESS — deleted once the measurement is recorded. +# +# The third producer state for the axe verdict. Runs 30910908474 (producer +# SKIPPED) and the validator positive control cover the other two; this one +# turns Playwright ON in a repo that is not a Nextcloud app, so the producer +# FAILS and uploads nothing. Expected: `Hydra Gates` still runs, the gates +# still report in full, and the job fails quoting playwright's resolved state +# as 'failure' rather than 'skipped'. +name: probe-axe-producer-failed + +on: + push: + branches: [fix/hydra-gates-needs-skip-deletion] + +permissions: + contents: write + actions: write + issues: write + pull-requests: write + +jobs: + arm: + uses: ./.github/workflows/quality.yml + with: + app-name: probe + enable-hydra-gates: true + enable-axe: true + enable-playwright: true + enable-phpunit: false + enable-newman: false + enable-frontend: false + enable-eslint: false + enable-psalm: false + enable-phpstan: false + enable-phpmetrics: false + enable-sbom: false + enable-coverage-guard: false From 086c917d278965105170aff3c267c50ee869c871 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Aug 2026 15:02:59 +0200 Subject: [PATCH 9/9] docs(quality): record the third-state attempt and why it collapsed into a skip --- .../workflows/_probe-axe-producer-failed.yml | 37 ------------------- .github/workflows/quality.yml | 11 ++++++ 2 files changed, 11 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/_probe-axe-producer-failed.yml diff --git a/.github/workflows/_probe-axe-producer-failed.yml b/.github/workflows/_probe-axe-producer-failed.yml deleted file mode 100644 index 23bb480..0000000 --- a/.github/workflows/_probe-axe-producer-failed.yml +++ /dev/null @@ -1,37 +0,0 @@ -# TEMPORARY MEASUREMENT HARNESS — deleted once the measurement is recorded. -# -# The third producer state for the axe verdict. Runs 30910908474 (producer -# SKIPPED) and the validator positive control cover the other two; this one -# turns Playwright ON in a repo that is not a Nextcloud app, so the producer -# FAILS and uploads nothing. Expected: `Hydra Gates` still runs, the gates -# still report in full, and the job fails quoting playwright's resolved state -# as 'failure' rather than 'skipped'. -name: probe-axe-producer-failed - -on: - push: - branches: [fix/hydra-gates-needs-skip-deletion] - -permissions: - contents: write - actions: write - issues: write - pull-requests: write - -jobs: - arm: - uses: ./.github/workflows/quality.yml - with: - app-name: probe - enable-hydra-gates: true - enable-axe: true - enable-playwright: true - enable-phpunit: false - enable-newman: false - enable-frontend: false - enable-eslint: false - enable-psalm: false - enable-phpstan: false - enable-phpmetrics: false - enable-sbom: false - enable-coverage-guard: false diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index e424dce..35b4aec 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -3354,6 +3354,17 @@ jobs: # only repo with the gates enabled — run 30911069404, Hydra Gates SUCCESS # with both axe steps SKIPPED. Nothing that is green today turns red. # + # Not measured end to end, and worth knowing why: the case where + # Playwright RUNS AND FAILS. Attempted in run 30911499894 by turning + # `enable-playwright` on, but this repo is not a Nextcloud app, so the + # security jobs failed first and the Playwright job's own + # `needs.security.result != 'failure'` guard turned it back into a skip. + # The gates job failed correctly there anyway, because what this step + # branches on is whether a usable report exists, not how the producer + # resolved — the producer's state appears only in the error text. That + # the gates job SURVIVES a genuinely failed producer is the thing that + # needed measuring, and run 30909727531 measured it directly. + # # The failure is RECORDED here and RAISED at the end of the job, not # thrown from this step. Measured first as a plain `exit 1` (run # 30910345294): it aborted the job before `Run the Hydra gates`, so a