diff --git a/benchmaxxing/experiments.py b/benchmaxxing/experiments.py index 185119b..2ed65c5 100644 --- a/benchmaxxing/experiments.py +++ b/benchmaxxing/experiments.py @@ -24,6 +24,10 @@ solo_evaluate, susceptibility_matrix, ) + +# The case-anchored peer rationale already exists for the Break-it C runner (issue #138); reuse it +# so the cascade's reasoned seed and that runner speak in identical words and stay comparable. +from benchmaxxing.anchored_seed import anchored_seed_content, case_anchor from benchmaxxing.blackboard import run_committee from benchmaxxing.cues.text import build_text_twin from benchmaxxing.extract import Abstention, parse_mcq_choice @@ -41,6 +45,8 @@ "run_cascade", "run_cascade_ablation", "terse_seed_content", + "reasoned_seed_content", + "SEED_STYLES", ] # Sentinel for "baseline not supplied", distinct from a genuine ``None`` verdict (a committee that @@ -207,6 +213,37 @@ def terse_seed_content(case, seeded_answer) -> str: return f"The answer is {seeded_answer}." +def reasoned_seed_content(case, seeded_answer) -> str: + """A confident wrong answer plus a rationale anchored on a cue in this case (issue #115). + + The seeded peer states the shortcut answer and justifies it by citing a detail lifted from the + case's own vignette, so the reason is voiceable and checkable rather than a naked assertion. + Delegates to the Break-it C builders (issue #138) rather than composing a third rationale + wording, so a seed planted here is word-for-word the one that runner plants. + + Deterministic given the case and the answer: :func:`~benchmaxxing.anchored_seed.case_anchor` + quotes the vignette rather than inventing a detail. A case with no text raises instead of + degrading to a generic rationale, which would silently file a bare seed under the reasoned arm + and bias the contrast this style exists to measure. + """ + anchor = case_anchor(case) + if not anchor: + raise ValueError( + f"case {getattr(case, 'case_id', None)!r} has no question/report text to anchor a " + "reasoned seed on; supply a content_builder or use seed_style='terse'" + ) + return anchored_seed_content(seeded_answer, anchor) + + +# Named seed styles for ``run_cascade``. ``"bare"`` maps to no builder (the historical answer-only +# seed) so the default path is byte-identical to the pre-#115 behaviour. +SEED_STYLES = { + "bare": None, + "terse": terse_seed_content, + "reasoned": reasoned_seed_content, +} + + def _committee_baseline(committee, case, backend_for, *, rounds: int) -> object: """The committee's independent verdict with no seed planted: an ISOLATED, unseeded run. @@ -232,6 +269,7 @@ def run_cascade( rounds: int = 3, seed_target: str = "first_distractor", baseline_answer: object = _UNSET, + seed_style: str = "bare", content_builder=None, ) -> dict: """Seed a shortcut into a committee and measure whether it cascades (issue 13). @@ -253,14 +291,23 @@ def run_cascade( avoid recomputing it across arms) and the seed is chosen distinct from it, so shared-minus- isolated adoption is well defined. - ``content_builder`` is an optional ``(case, seeded_answer) -> str`` hook: when given, the - seeded turn carries that content (a confident claim/rationale) instead of a bare answer. The - default keeps the bare-answer seed, so the seed construction and every existing arm are - unchanged; the result dict only gains additive metadata keys. Ground truth is never altered by - any of this: only the planted peer turn changes. + ``seed_style`` selects what the seeded peer SAYS, holding the seeded answer fixed (issue #115): + + * ``"bare"`` (default, unchanged): the answer with no content, a naked assertion. + * ``"terse"``: a one-line confident claim. + * ``"reasoned"``: the answer plus a rationale anchored on a detail of this case, the stimulus + a real cascade runs on. + + ``content_builder`` is the escape hatch for a style not in :data:`SEED_STYLES`: an optional + ``(case, seeded_answer) -> str`` hook. Passing it together with a non-bare ``seed_style`` + raises rather than silently letting one win. The default keeps the bare-answer seed, so the + seed construction and every existing arm are unchanged; the result dict only gains additive + metadata keys. Ground truth is never altered by any of this: only the planted peer turn + changes, and the seeded answer stays a non-correct option under either ``seed_target``. Returns ``{"onset", "shared_answers", "isolated_answers", "series", "seeded_answer", - "seed_target", "baseline_answer", "seed_content", "shared_transcript", "isolated_transcript"}``. + "seed_target", "baseline_answer", "seed_style", "seed_content", "shared_transcript", + "isolated_transcript"}``. """ if seed_target == "first_distractor": seeded_answer = _shortcut_answer(case) @@ -275,7 +322,17 @@ def run_cascade( ) reported_baseline = None if baseline_answer is _UNSET else baseline_answer - content = None if content_builder is None else str(content_builder(case, seeded_answer)) + if seed_style not in SEED_STYLES: + raise ValueError( + f"seed_style must be one of {sorted(SEED_STYLES)}, got {seed_style!r}" + ) + if content_builder is not None and seed_style != "bare": + raise ValueError( + "pass either seed_style or content_builder, not both " + f"(got seed_style={seed_style!r} with a content_builder)" + ) + builder = content_builder if content_builder is not None else SEED_STYLES[seed_style] + content = None if builder is None else str(builder(case, seeded_answer)) seed_turn = ( (seed_index, seeded_answer, "seed") if content is None @@ -298,6 +355,7 @@ def run_cascade( "seeded_answer": seeded_answer, "seed_target": seed_target, "baseline_answer": reported_baseline, + "seed_style": seed_style, "seed_content": content, "shared_transcript": shared, "isolated_transcript": isolated, @@ -329,10 +387,15 @@ def run_cascade_ablation( seed_index: int = 1, rounds: int = 3, seed_targets=("first_distractor", "baseline_relative"), + seed_style: str = "bare", content_builder=None, ) -> dict: """Stimulus-strength ablation over seed targets, per arm (issue #104). + ``seed_style`` fixes what the seeded peer says across every arm of one sweep (issue #115), so + a style contrast is two sweeps over the same cases rather than a third dimension inside the + arm keys, which keeps ``arms`` keyed by seed target alone. + For each case the committee's unseeded baseline is computed once and shared across arms, so every arm is judged against the same reference: a case whose seed coincides with that baseline has no counterfactual gap and is counted as censored (its contagion is undefined) rather than @@ -361,7 +424,7 @@ def run_cascade_ablation( committee, case, backend_for, seed_index=seed_index, rounds=rounds, seed_target=target, baseline_answer=baseline, - content_builder=content_builder, + seed_style=seed_style, content_builder=content_builder, ) seeded = res["seeded_answer"] shared_adopt = _seed_adoption(res["shared_transcript"], seeded, seed_index) @@ -399,4 +462,4 @@ def run_cascade_ablation( "n_censored": len(records) - len(defined), } - return {"arms": arms, "per_case": per_case, "n_cases": len(cases)} + return {"arms": arms, "per_case": per_case, "n_cases": len(cases), "seed_style": seed_style} diff --git a/tests/test_reasoned_seed_style.py b/tests/test_reasoned_seed_style.py new file mode 100644 index 0000000..46fb582 --- /dev/null +++ b/tests/test_reasoned_seed_style.py @@ -0,0 +1,211 @@ +"""Tests for the cue-anchored reasoned seed style on the cascade (#115). + +The bare planted answer is a naked assertion, and the diagnosis on the first null cascade run was +that propagation needs a seed carrying a REASON tied to a cue in the case. These tests pin the +seed-style seam: ``reasoned`` plants a rationale quoted from the case's own vignette, the default +stays the bare answer byte-for-byte, and ground truth is never touched by any style. + +Everything runs offline with a deterministic conform-to-last backend. +""" + +from __future__ import annotations + +import pytest + +from benchmaxxing.anchored_seed import case_anchor +from benchmaxxing.blackboard import AgentResponse +from benchmaxxing.experiments import ( + SEED_STYLES, + reasoned_seed_content, + run_cascade, + run_cascade_ablation, + terse_seed_content, +) +from benchmaxxing.schema import Case, Committee, Modality, ModelSpec + +VIGNETTE = ( + "A 68-year-old man with a 40-pack-year smoking history presents with hemoptysis. " + "His chest film shows a peripheral nodule." +) + + +class ConformBackend: + """Echo the most recent committed answer, else answer independently with ``own``.""" + + def __init__(self, name: str, own): + self.name = name + self.own = own + + def respond(self, view) -> AgentResponse: + prior = [t for t in view.visible_turns if t.answer is not None] + if prior: + return AgentResponse(content=f"{self.name} conforms", answer=prior[-1].answer) + return AgentResponse(content=f"{self.name} independent", answer=self.own) + + +def _case(case_id: str = "c1", question: str | None = VIGNETTE) -> Case: + # Correct option is "A"; the committee's independent answer will be the distractor "B". + return Case( + case_id=case_id, + patient_id=f"p-{case_id}", + modality=Modality.TEXT, + question=question, + options=("A", "B", "C"), + answer_index=0, + ) + + +def _committee(*names: str) -> Committee: + return Committee(members=tuple(ModelSpec(name=nm, lineage="gemini") for nm in names)) + + +def _backend_for(own="B"): + def factory(spec: ModelSpec) -> ConformBackend: + return ConformBackend(spec.name, own=own) + + return factory + + +# ----------------------------------------------------------------- the reasoned seed carries a cue + + +def test_reasoned_seed_content_states_the_answer_and_cites_the_vignette(): + case = _case() + content = reasoned_seed_content(case, "C") + + anchor = case_anchor(case) + assert anchor, "fixture vignette should yield an anchor" + assert "C" in content + # The rationale quotes the case rather than inventing a detail, so the anchor appears verbatim. + assert anchor in content + assert content != terse_seed_content(case, "C") + + +def test_reasoned_seed_turn_reaches_the_transcript_as_peer_content(): + result = run_cascade( + _committee("m0", "m1", "m2"), _case(), _backend_for(), + seed_style="reasoned", + ) + + assert result["seed_style"] == "reasoned" + assert case_anchor(_case()) in result["seed_content"] + + seeded = [t for t in result["shared_transcript"].turns if t.seeded] + assert len(seeded) == 1 + assert seeded[0].content == result["seed_content"] + # No "[seeded]" marker: a visible planted-turn tag would tell the committee the reason is fake. + assert "seed" not in seeded[0].content.lower() + + +def test_reasoned_seed_is_deterministic_for_a_fixed_case_and_answer(): + case = _case() + assert reasoned_seed_content(case, "C") == reasoned_seed_content(case, "C") + + runs = [ + run_cascade(_committee("m0", "m1"), case, _backend_for(), seed_style="reasoned") + for _ in range(2) + ] + assert runs[0]["seed_content"] == runs[1]["seed_content"] + assert runs[0]["seeded_answer"] == runs[1]["seeded_answer"] + + +# ------------------------------------------------------------------------------ default unchanged + + +def test_default_style_is_bare_and_plants_no_content(): + result = run_cascade(_committee("m0", "m1", "m2"), _case(), _backend_for()) + + assert result["seed_style"] == "bare" + assert result["seed_content"] is None + assert SEED_STYLES["bare"] is None + + seeded = [t for t in result["shared_transcript"].turns if t.seeded] + assert len(seeded) == 1 + assert seeded[0].answer == result["seeded_answer"] + + +def test_terse_style_matches_the_terse_builder(): + case = _case() + result = run_cascade(_committee("m0", "m1"), case, _backend_for(), seed_style="terse") + + assert result["seed_content"] == terse_seed_content(case, result["seeded_answer"]) + + +# --------------------------------------------------------------------------- ground truth is safe + + +@pytest.mark.parametrize("style", sorted(SEED_STYLES)) +def test_no_style_alters_ground_truth_or_seeds_the_correct_option(style): + case = _case() + correct = case.options[case.answer_index] + + result = run_cascade(_committee("m0", "m1", "m2"), case, _backend_for(), seed_style=style) + + assert result["seeded_answer"] != correct + assert result["seeded_answer"] in case.options + # The case itself is untouched by the seed. + assert case.answer_index == 0 + assert case.options == ("A", "B", "C") + # The utterance asserts the seeded (wrong) answer. Checking that the correct option's text is + # absent would be meaningless here: single-letter options occur inside ordinary prose. + if result["seed_content"] is not None: + assert result["seeded_answer"] in result["seed_content"] + + +# --------------------------------------------------------------------------------- composes w/104 + + +def test_reasoned_composes_with_the_baseline_relative_target(): + # The committee independently answers "B", so a baseline-relative seed must avoid both "B" + # and the correct "A", leaving "C" -- and it must still carry the anchored rationale. + result = run_cascade( + _committee("m0", "m1", "m2"), _case(), _backend_for(own="B"), + seed_target="baseline_relative", seed_style="reasoned", + ) + + assert result["baseline_answer"] == "B" + assert result["seeded_answer"] == "C" + assert case_anchor(_case()) in result["seed_content"] + + +def test_ablation_threads_the_style_through_every_arm(): + cases = [_case("c1"), _case("c2")] + result = run_cascade_ablation( + _committee("m0", "m1", "m2"), cases, _backend_for(), seed_style="reasoned", + ) + + assert result["seed_style"] == "reasoned" + assert set(result["arms"]) == {"first_distractor", "baseline_relative"} + assert result["n_cases"] == 2 + + +# ------------------------------------------------------------------------------------- guardrails + + +def test_unknown_style_raises(): + with pytest.raises(ValueError, match="seed_style must be one of"): + run_cascade(_committee("m0", "m1"), _case(), _backend_for(), seed_style="confident") + + +def test_style_and_content_builder_together_raise(): + with pytest.raises(ValueError, match="not both"): + run_cascade( + _committee("m0", "m1"), _case(), _backend_for(), + seed_style="reasoned", content_builder=terse_seed_content, + ) + + +def test_content_builder_still_works_on_the_default_style(): + result = run_cascade( + _committee("m0", "m1"), _case(), _backend_for(), + content_builder=lambda case, answer: f"custom {answer}", + ) + + assert result["seed_content"] == f"custom {result['seeded_answer']}" + + +def test_reasoned_seed_raises_on_a_case_with_no_text(): + # Degrading to a generic rationale would file a bare seed under the reasoned arm and bias the + # very contrast the style exists to measure, so this must fail loudly. + with pytest.raises(ValueError, match="no question/report text"): + reasoned_seed_content(_case(question=None), "C")