From 1e5d0adeddbd3c6bbeb38b1e6f2a41dcc10091e2 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Sun, 20 Sep 2026 10:20:39 +0200 Subject: [PATCH] cmake: let a red leg name every failing test, not just the first (fixes #616) All twelve test presets set `stopOnFailure: true`, so `ctest` abandoned the run at the first failure on every CI leg. One broken test and thirty broken tests therefore produced the same log, and every diagnosis made from that log was made on partial information. That cost is not hypothetical: CI on #585's head was read as "all six failing jobs fail on one test", and the singularity was treated as a property of the defect. Six tests were failing; `ctest` stopped at the first, and it took a full local build by someone else to establish that. Decision, per preset: **removed on all twelve.** Each ctest case is its own process here (`catch_discover_tests`), so no leg's later tests are invalidated by an earlier failure -- not the sanitizer legs, where a diagnostic aborts only its own process, and not `clang-coverage`, where stopping early loses profile data rather than protecting it. The "it saves time" argument does not survive the matrix either: the leg that already reports in full is Valgrind, the slowest one there is. A leg that ever does earn the setting can set it on its own preset, with its reason in its own `description`. The twelve presets now inherit a hidden `base-test` preset that carries `output`/`execution` once, so the decision lives in exactly one place with the reasoning next to it, instead of twelve copies of a setting nobody recorded a reason for. `ctest` has no `--no-stop-on-failure`, so keeping it in the preset and overriding it per invocation in CI is not available -- the preset is the only place the choice can be made. No workflow change is needed: every leg runs its suite through `ctest --preset ...`. Measured, gcc-debug on this revision, with two Catch2 cases in tests/test_rational.cpp deliberately broken (reverted before this commit) and the run filtered to `-R "^Rational::"`: # before, stopOnFailure: true 2/46 Test #597: Rational::Comparison ......***Failed 0.00 sec 50% tests passed, 1 tests failed out of 2 The following tests FAILED: 597 - Rational::Comparison (Failed) # after 46/46 Test #641: Rational::toDouble::PrecisionRoundtrip ... Passed 0.00 sec 96% tests passed, 2 tests failed out of 46 The following tests FAILED: 597 - Rational::Comparison (Failed) 599 - Rational::Constants (Failed) Both runs exit 8. Restoring `stopOnFailure` on the inheriting `gcc-debug` preset alone reproduces the one-line report, which is what shows the setting and not something else is responsible. Inheritance was checked rather than assumed. `noTestsAction: error` still reaches each leg through `base-test`: `ctest --preset gcc-debug -R zzz_no_such_test` exits 8 where the same filter without the preset exits 0. `outputOnFailure` likewise -- both failing cases printed their Catch2 bodies in the "after" run above. And CMake merges `execution` key by key across `inherits`: the child that set only `stopOnFailure` still errored on no tests. Full suite green on the new presets: 100% tests passed out of 1562 (gcc-debug, GNU 16.2.1). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AbwhcguQFkhvVi2AH19sWk --- CMakePresets.json | 87 +++++++++-------------------------------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 7fb70c771..e4bc649ac 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -226,77 +226,24 @@ ], "testPresets": [ { - "name": "cl-debug", - "configurePreset": "cl-debug", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "cl-qt-debug", - "configurePreset": "cl-qt-debug", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "cl-qt-release", - "configurePreset": "cl-qt-release", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clangcl-debug", - "configurePreset": "clangcl-debug", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "windows-everything", - "configurePreset": "windows-everything", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "gcc-debug", - "configurePreset": "gcc-debug", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clang-debug", - "configurePreset": "clang-debug", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "linux-everything", - "configurePreset": "linux-everything", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clang-asan", - "configurePreset": "clang-asan", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clang-tsan", - "configurePreset": "clang-tsan", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clang-ubsan", - "configurePreset": "clang-ubsan", - "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - }, - { - "name": "clang-coverage", - "configurePreset": "clang-coverage", + "name": "base-test", + "hidden": true, + "description": "Shared test-run policy for every leg. Two things a reader needs. (1) stopOnFailure is deliberately NOT set here. A red CI leg has to name every test that failed, not whichever one ctest happened to reach first: stopping early makes one broken test and thirty broken tests look identical in the log, and a diagnosis made from that log is made on partial information (morph#616). ctest has no --no-stop-on-failure, so 'set it in the preset and override it in CI' is not available -- the preset is the only place the decision can be made. (2) A leg whose later tests are genuinely meaningless after an early failure may set stopOnFailure on its own preset, but must say why in its own description. Inheritance merges 'execution' key by key -- measured on CMake 4.4.3: a child preset that sets only stopOnFailure still gets noTestsAction from here -- so setting it back is a one-key change, not a copy of this whole block.", "output": { "outputOnFailure": true }, - "execution": { "noTestsAction": "error", "stopOnFailure": true } - } + "execution": { "noTestsAction": "error" } + }, + { "name": "cl-debug", "configurePreset": "cl-debug", "inherits": "base-test" }, + { "name": "cl-qt-debug", "configurePreset": "cl-qt-debug", "inherits": "base-test" }, + { "name": "cl-qt-release", "configurePreset": "cl-qt-release", "inherits": "base-test" }, + { "name": "clangcl-debug", "configurePreset": "clangcl-debug", "inherits": "base-test" }, + { "name": "windows-everything","configurePreset": "windows-everything","inherits": "base-test" }, + { "name": "gcc-debug", "configurePreset": "gcc-debug", "inherits": "base-test" }, + { "name": "clang-debug", "configurePreset": "clang-debug", "inherits": "base-test" }, + { "name": "linux-everything", "configurePreset": "linux-everything", "inherits": "base-test" }, + { "name": "clang-asan", "configurePreset": "clang-asan", "inherits": "base-test" }, + { "name": "clang-tsan", "configurePreset": "clang-tsan", "inherits": "base-test" }, + { "name": "clang-ubsan", "configurePreset": "clang-ubsan", "inherits": "base-test" }, + { "name": "clang-coverage", "configurePreset": "clang-coverage", "inherits": "base-test" } ], "workflowPresets": [ {