From 1e038dc046ff1ebdb49ba7cf0a500e821ced664a Mon Sep 17 00:00:00 2001 From: Naser Derakhshan <9349840+nasser1941@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:52:40 +0100 Subject: [PATCH] Don't count a failing test run piped into tail as passed `pytest -q | tail` exits with tail's 0 when tests fail. lcode read that as the checks passing, didn't run them, and reported them as passed. A run whose exit code isn't the check's now counts only if its output shows a test summary with nothing failing. Otherwise lcode runs the checks itself. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +++++ docs/usage.md | 2 ++ src/lcode/agent_checks.py | 2 +- src/lcode/checks.py | 18 ++++++++++ tests/test_checks.py | 69 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22694f0..2b45d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to lcode are documented here. The format follows ## [Unreleased] +### Fixed + +- **A failing test run piped into `tail` counted as passing.** When the model ran the project's checks + itself as `pytest -q | tail`, lcode read `tail`'s exit code of 0, took the checks as passed and didn't + run them, and `lcode -p --output json` reported them as passed. A run whose exit code isn't the check's + now counts only if its output shows a test summary with nothing failing. Otherwise lcode runs the + checks itself. + ## [0.61.0] - 2026-11-05 ### Added diff --git a/docs/usage.md b/docs/usage.md index 2583f6d..133e3bc 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -227,6 +227,8 @@ runs the project's checks itself before the answer is final: - **When nothing could run** (no tests collected, or the test runner isn't installed), lcode says so and doesn't send it to the model. - **When the model already ran them** after its last change and they passed, they aren't run again. + A run piped into another program (`pytest -q | tail`) exits with that program's code, so it counts + only if its output shows a test summary with nothing failing. - **Permission:** they run like any other command, so in `ask` mode the first run asks (++a++ allows them for the session). - **Programs** using `lcode -p --output json` see the runs in the result's `checks`. diff --git a/src/lcode/agent_checks.py b/src/lcode/agent_checks.py index aba7ba7..859bbfb 100644 --- a/src/lcode/agent_checks.py +++ b/src/lcode/agent_checks.py @@ -115,7 +115,7 @@ def project_check(self) -> checks.Check | None: def model_ran_checks(self, command: str, result: str) -> bool: """Whether a command the model ran was the project's checks, and they passed.""" check = self.project_check() - if check is None or not checks.covers(check, command) or checks.exit_code(result) != 0: + if check is None or not checks.covers(check, command) or not checks.run_passed(check, command, result): return False self.check_results.append({"command": check.command, "exit_code": 0, "by": "model"}) return True diff --git a/src/lcode/checks.py b/src/lcode/checks.py index cdc095f..4031d3e 100644 --- a/src/lcode/checks.py +++ b/src/lcode/checks.py @@ -165,6 +165,24 @@ def covers(check: Check, command: str) -> bool: return " ".join(check.command.split()) in " ".join(command.split()) +KEEPS_EXIT_CODE = re.compile(r"\d*>&\d+|&>>?|&&") # redirections and `&&`, after the check +HIDES_EXIT_CODE = re.compile(r"[|;&\n]") # a pipe, `||`, `;`, `&` or another line: the exit code isn't the check's +RUN_FAILED = re.compile(r"\b[1-9]\d* (?:failed|errors?)\b|^(?:FAILED|ERROR)\b", re.M) +RUN_PASSED = re.compile(r"\b[1-9]\d* passed\b|^OK\b", re.M) + + +def run_passed(check: Check, command: str, result: str) -> bool: + """Whether the model's own run of the check passed. `pytest -q | tail` exits with tail's 0 when tests fail, so when + the exit code isn't the check's, only the output can say: a test runner's summary, with nothing failing.""" + if exit_code(result) != 0: + return False + found = re.search(r"\s+".join(map(re.escape, check.command.split())), command) + after = KEEPS_EXIT_CODE.sub(" ", command[found.end() :]).rstrip() if found else "" + if not HIDES_EXIT_CODE.search(after): + return True + return RUN_PASSED.search(result) is not None and RUN_FAILED.search(result) is None + + def failure_note(check: Check, result: str, round_: int) -> str: output = result if len(result) <= OUTPUT_TAIL else "…" + result[-OUTPUT_TAIL:] last = round_ >= MAX_ROUNDS diff --git a/tests/test_checks.py b/tests/test_checks.py index b356ff8..47ae51a 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -100,18 +100,79 @@ def test_failing_checks_go_back_to_the_model_twice_at_most(make_agent): assert "Checks still fail" in text(agent) and [r["exit_code"] for r in agent.check_results] == [1, 1] -def test_checks_the_model_ran_after_its_change_arent_repeated(make_agent): +@pytest.mark.parametrize( + ("run", "command"), + [ + (PASS, "cd . && {check} 2>&1"), # the exit code is the check's + ("python3 -c \"print('3 passed in 0.01s')\"", "cd . && {check} 2>&1 | tail -5"), # the output says so + ], +) +def test_checks_the_model_ran_after_its_change_arent_repeated(make_agent, run, command): agent = make_agent( [ reply(tool_calls=[call("write_file", path="calc.py", content="x = 1\n")]), - reply(tool_calls=[call("bash", command=f"cd . && {PASS} 2>&1 | tail -5")]), + reply(tool_calls=[call("bash", command=command.format(check=run))]), reply("Done, the tests pass."), ], - checks=PASS, + checks=run, ) agent.run_turn("fix calc") assert "Checking the work" not in output(agent) - assert agent.check_results == [{"command": PASS, "exit_code": 0, "by": "model"}] + assert agent.check_results == [{"command": run, "exit_code": 0, "by": "model"}] + + +def test_a_failing_run_piped_into_tail_doesnt_count_as_passed(make_agent): + agent = make_agent( + [ + reply(tool_calls=[call("write_file", path="calc.py", content="x = 1\n")]), + reply(tool_calls=[call("bash", command=f"{FAIL} 2>&1 | tail -5")]), # exits with tail's 0 + reply("Done, the tests pass."), + reply("The test still fails: it expects 4."), + ], + checks=FAIL, + ) + agent.run_turn("fix calc") + assert "◆ Checking the work" in text(agent) + assert agent.check_results == [{"command": FAIL, "exit_code": 1, "by": "lcode"}] + notes = [m["content"] for m in agent.messages if m["role"] == "user" and m["content"].startswith("[lcode]")] + assert len(notes) == 1 and "FAIL: test_average" in notes[0] + + +@pytest.mark.parametrize( + ("command", "shown", "passed"), + [ + ("python -m pytest -q", "18 passed in 3.2s", True), + ("cd x && python -m pytest -q 2>&1", "no summary in this output", True), + ("python -m pytest -q > log.txt 2>&1", "", True), + ("python -m pytest -q &> log.txt", "", True), + ("python -m pytest -q && echo done", "done", True), + ("python -m pytest -q 2>&1 | tail -5", "18 passed in 3.2s", True), + ("python -m pytest -q | tail -3", "12 passed, 0 failed", True), + ("python -m pytest -q | tee log.txt", "Ran 3 tests in 0.1s\n\nOK", True), + ("python -m pytest -q 2>&1 | tail -5", "1 failed, 17 passed in 3.6s", False), + ("python -m pytest -q 2>&1 | tail -5", "2 errors in 0.5s", False), + ("python -m pytest -q 2>&1 | tail -5", "5 passed, 1 error in 0.5s", False), + ("python -m pytest -q | tail -3", "FAILED tests/test_a.py::test_b\n3 passed", False), + ("python -m pytest -q | tail -3", "ERROR tests/test_a.py\n3 passed", False), + ("python -m pytest -q | tail -3", "Ran 3 tests in 0.1s\n\nFAILED (failures=1)", False), + ("python -m pytest -q | head -5", "collecting ...", False), # the summary was cut off + ("python -m pytest -q 2>&1 | tail -5", "no tests ran in 0.1s", False), + ("python -m pytest -q || true", "1 failed in 0.1s", False), + ("python -m pytest -q; echo done", "1 failed in 0.1s\ndone", False), + ("python -m pytest -q\necho done", "1 failed in 0.1s\ndone", False), + ("python -m pytest -q & wait", "", False), + ], +) +def test_a_run_passed_when_its_exit_code_or_output_says_so(command, shown, passed): + check = checks.Check("python -m pytest -q", "tests/") + assert checks.covers(check, command) + assert checks.run_passed(check, command, f"{shown}\n[exit code: 0]") is passed + + +def test_a_run_that_exits_with_an_error_never_passed(): + check = checks.Check("python -m pytest -q", "tests/") + assert not checks.run_passed(check, "set -o pipefail; python -m pytest -q | tail", "3 passed\n[exit code: 1]") + assert not checks.run_passed(check, "python -m pytest -q", "the command didn't run") def test_no_checks_without_changes_or_in_plan_mode(make_agent):