Don't count a failing test run piped into tail as passed

CHANGELOG.md

@@ -6,6 +6,14 @@ All notable changes to lcode are documented here. The format follows
 
 ## [Unreleased]
 
+### Fixed
+
+- **A failing test run piped into `tail` counted as passing.** When the model ran the project's checks
+  itself as `pytest -q | tail`, lcode read `tail`'s exit code of 0, took the checks as passed and didn't
+  run them, and `lcode -p --output json` reported them as passed. A run whose exit code isn't the check's
+  now counts only if its output shows a test summary with nothing failing. Otherwise lcode runs the
+  checks itself.
+
 ## [0.61.0] - 2026-11-05
 
 ### Added

docs/usage.md

@@ -227,6 +227,8 @@ runs the project's checks itself before the answer is final:
 - **When nothing could run** (no tests collected, or the test runner isn't installed), lcode says so
   and doesn't send it to the model.
 - **When the model already ran them** after its last change and they passed, they aren't run again.
+  A run piped into another program (`pytest -q | tail`) exits with that program's code, so it counts
+  only if its output shows a test summary with nothing failing.
 - **Permission:** they run like any other command, so in `ask` mode the first run asks (++a++ allows
   them for the session).
 - **Programs** using `lcode -p --output json` see the runs in the result's `checks`.

src/lcode/agent_checks.py

@@ -115,7 +115,7 @@ def project_check(self) -> checks.Check | None:
     def model_ran_checks(self, command: str, result: str) -> bool:
         """Whether a command the model ran was the project's checks, and they passed."""
         check = self.project_check()
-        if check is None or not checks.covers(check, command) or checks.exit_code(result) != 0:
+        if check is None or not checks.covers(check, command) or not checks.run_passed(check, command, result):
             return False
         self.check_results.append({"command": check.command, "exit_code": 0, "by": "model"})
         return True

src/lcode/checks.py

@@ -165,6 +165,24 @@ def covers(check: Check, command: str) -> bool:
     return " ".join(check.command.split()) in " ".join(command.split())
 
 
+KEEPS_EXIT_CODE = re.compile(r"\d*>&\d+|&>>?|&&")  # redirections and `&&`, after the check
+HIDES_EXIT_CODE = re.compile(r"[|;&\n]")  # a pipe, `||`, `;`, `&` or another line: the exit code isn't the check's
+RUN_FAILED = re.compile(r"\b[1-9]\d* (?:failed|errors?)\b|^(?:FAILED|ERROR)\b", re.M)
+RUN_PASSED = re.compile(r"\b[1-9]\d* passed\b|^OK\b", re.M)
+
+
+def run_passed(check: Check, command: str, result: str) -> bool:
+    """Whether the model's own run of the check passed. `pytest -q | tail` exits with tail's 0 when tests fail, so when
+    the exit code isn't the check's, only the output can say: a test runner's summary, with nothing failing."""
+    if exit_code(result) != 0:
+        return False
+    found = re.search(r"\s+".join(map(re.escape, check.command.split())), command)
+    after = KEEPS_EXIT_CODE.sub(" ", command[found.end() :]).rstrip() if found else ""
+    if not HIDES_EXIT_CODE.search(after):
+        return True
+    return RUN_PASSED.search(result) is not None and RUN_FAILED.search(result) is None
+
+
 def failure_note(check: Check, result: str, round_: int) -> str:
     output = result if len(result) <= OUTPUT_TAIL else "…" + result[-OUTPUT_TAIL:]
     last = round_ >= MAX_ROUNDS

tests/test_checks.py

@@ -100,18 +100,79 @@ def test_failing_checks_go_back_to_the_model_twice_at_most(make_agent):
     assert "Checks still fail" in text(agent) and [r["exit_code"] for r in agent.check_results] == [1, 1]
 
 
-def test_checks_the_model_ran_after_its_change_arent_repeated(make_agent):
+@pytest.mark.parametrize(
+    ("run", "command"),
+    [
+        (PASS, "cd . && {check} 2>&1"),  # the exit code is the check's
+        ("python3 -c \"print('3 passed in 0.01s')\"", "cd . && {check} 2>&1 | tail -5"),  # the output says so
+    ],
+)
+def test_checks_the_model_ran_after_its_change_arent_repeated(make_agent, run, command):
     agent = make_agent(
         [
             reply(tool_calls=[call("write_file", path="calc.py", content="x = 1\n")]),
-            reply(tool_calls=[call("bash", command=f"cd . && {PASS} 2>&1 | tail -5")]),
+            reply(tool_calls=[call("bash", command=command.format(check=run))]),
             reply("Done, the tests pass."),
         ],
-        checks=PASS,
+        checks=run,
     )
     agent.run_turn("fix calc")
     assert "Checking the work" not in output(agent)
-    assert agent.check_results == [{"command": PASS, "exit_code": 0, "by": "model"}]
+    assert agent.check_results == [{"command": run, "exit_code": 0, "by": "model"}]
+
+
+def test_a_failing_run_piped_into_tail_doesnt_count_as_passed(make_agent):
+    agent = make_agent(
+        [
+            reply(tool_calls=[call("write_file", path="calc.py", content="x = 1\n")]),
+            reply(tool_calls=[call("bash", command=f"{FAIL} 2>&1 | tail -5")]),  # exits with tail's 0
+            reply("Done, the tests pass."),
+            reply("The test still fails: it expects 4."),
+        ],
+        checks=FAIL,
+    )
+    agent.run_turn("fix calc")
+    assert "◆ Checking the work" in text(agent)
+    assert agent.check_results == [{"command": FAIL, "exit_code": 1, "by": "lcode"}]
+    notes = [m["content"] for m in agent.messages if m["role"] == "user" and m["content"].startswith("[lcode]")]
+    assert len(notes) == 1 and "FAIL: test_average" in notes[0]
+
+
+@pytest.mark.parametrize(
+    ("command", "shown", "passed"),
+    [
+        ("python -m pytest -q", "18 passed in 3.2s", True),
+        ("cd x && python -m pytest -q 2>&1", "no summary in this output", True),
+        ("python  -m pytest   -q > log.txt 2>&1", "", True),
+        ("python -m pytest -q &> log.txt", "", True),
+        ("python -m pytest -q && echo done", "done", True),
+        ("python -m pytest -q 2>&1 | tail -5", "18 passed in 3.2s", True),
+        ("python -m pytest -q | tail -3", "12 passed, 0 failed", True),
+        ("python -m pytest -q | tee log.txt", "Ran 3 tests in 0.1s\n\nOK", True),
+        ("python -m pytest -q 2>&1 | tail -5", "1 failed, 17 passed in 3.6s", False),
+        ("python -m pytest -q 2>&1 | tail -5", "2 errors in 0.5s", False),
+        ("python -m pytest -q 2>&1 | tail -5", "5 passed, 1 error in 0.5s", False),
+        ("python -m pytest -q | tail -3", "FAILED tests/test_a.py::test_b\n3 passed", False),
+        ("python -m pytest -q | tail -3", "ERROR tests/test_a.py\n3 passed", False),
+        ("python -m pytest -q | tail -3", "Ran 3 tests in 0.1s\n\nFAILED (failures=1)", False),
+        ("python -m pytest -q | head -5", "collecting ...", False),  # the summary was cut off
+        ("python -m pytest -q 2>&1 | tail -5", "no tests ran in 0.1s", False),
+        ("python -m pytest -q || true", "1 failed in 0.1s", False),
+        ("python -m pytest -q; echo done", "1 failed in 0.1s\ndone", False),
+        ("python -m pytest -q\necho done", "1 failed in 0.1s\ndone", False),
+        ("python -m pytest -q & wait", "", False),
+    ],
+)
+def test_a_run_passed_when_its_exit_code_or_output_says_so(command, shown, passed):
+    check = checks.Check("python -m pytest -q", "tests/")
+    assert checks.covers(check, command)
+    assert checks.run_passed(check, command, f"{shown}\n[exit code: 0]") is passed
+
+
+def test_a_run_that_exits_with_an_error_never_passed():
+    check = checks.Check("python -m pytest -q", "tests/")
+    assert not checks.run_passed(check, "set -o pipefail; python -m pytest -q | tail", "3 passed\n[exit code: 1]")
+    assert not checks.run_passed(check, "python -m pytest -q", "the command didn't run")
 
 
 def test_no_checks_without_changes_or_in_plan_mode(make_agent):