diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 147e81a2f..706d87a92 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -75,6 +75,7 @@ Expected folder layout: scenarios_data/ scenario_11/ groundtruth.txt + groundtruth_eval.json # optional scenario_12/ groundtruth.txt scenario_13/ @@ -95,6 +96,46 @@ scenario_11 → 11 The evaluator then joins trajectories and ground truth by scenario id. +### Optional evaluation metadata + +Scenario folders may also include `groundtruth_eval.json`. This file provides +scorer-specific metadata while keeping the canonical answer in `groundtruth.txt`. +It is currently used by clarification-abstain-response (CAR) scenarios, where +the exact wording can vary but the response mode and key domain terms must be +correct. + +Example `groundtruth.txt`: + +```json +{ + "clarification": "Which asset do you mean by 'the main unit'?" +} +``` + +Example `groundtruth_eval.json`: + +```json +{ + "mode": "clarification", + "required_terms": ["main unit"], + "optional_terms": ["asset"], + "must_have_exactly_one_mode_key": true +} +``` + +Supported CAR metadata fields: + +| Field | Meaning | +| -------------------------------- | ----------------------------------------------------------------------- | +| `mode` | Expected top-level key: `response`, `clarification`, or `abstain` | +| `required_terms` | Terms that indicate the model addressed the required ambiguity or fact | +| `optional_terms` | Informative terms reported for analysis, but not required for passing | +| `must_have_exactly_one_mode_key` | Whether the model must return exactly one top-level CAR mode key | + +If `groundtruth_eval.json` is absent, the scorer derives CAR metadata from +`groundtruth.txt` for one-key answers using `response`, `clarification`, or +`abstain`. + --- ## How Matching Works @@ -180,6 +221,11 @@ The scorer also handles simple noisy count-only answers such as: The answer is 34. ``` +For noisy structured answers, the parser prefers a final JSON/Python structure +over earlier explanatory parenthetical text. This avoids evaluating prose such +as `(no asset is specified)` when the model later returns the actual final JSON +object. + --- ## Metrics @@ -200,6 +246,45 @@ The answer is 34. The scorer marks a scenario as passed only when the structured answer is a strict exact match. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. +### CAR metrics and pass criteria + +For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus +required-term coverage. The model answer should be a single JSON object using +exactly one of these top-level keys: + +```text +response | clarification | abstain +``` + +A CAR scenario passes when: + +1. the model returns exactly one top-level CAR mode key, +2. that key matches the expected `mode`, and +3. at least one `required_terms` entry appears in the model answer value. + +The CAR soft score is reported as: + +```text +car_score = 0.6 * mode_key_match + 0.4 * mode_term_coverage +``` + +where `mode_term_coverage` is the fraction of required terms found in the answer +value. Optional terms are reported for diagnostics and do not affect pass/fail. + +Additional CAR report fields: + +| Metric | Meaning | +| ----------------------------------- | ------------------------------------------------------------------ | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required terms used for this scenario | +| `mode_matched_terms` | Required terms found in the model answer | +| `mode_term_coverage` | Fraction of required terms found | +| `mode_optional_terms` | Optional diagnostic terms | +| `mode_matched_optional_terms` | Optional diagnostic terms found | +| `mode_optional_term_coverage` | Fraction of optional terms found, or null if none were configured | +| `car_score` | Weighted CAR soft score | + --- ## Example diff --git a/src/evaluation/loader.py b/src/evaluation/loader.py index 04278e1a6..c8fbf6dd9 100644 --- a/src/evaluation/loader.py +++ b/src/evaluation/loader.py @@ -100,6 +100,12 @@ def _load_scenario_dir(path: Path) -> list[Scenario]: scenario_id = scenario_id.removeprefix("scenario_") expected_answer = groundtruth_path.read_text(encoding="utf-8").strip() + eval_metadata_path = child / "groundtruth_eval.json" + evaluation_metadata = None + if eval_metadata_path.exists(): + evaluation_metadata = json.loads( + eval_metadata_path.read_text(encoding="utf-8") + ) question_path = child / "question.txt" text = ( @@ -115,6 +121,7 @@ def _load_scenario_dir(path: Path) -> list[Scenario]: "text": text, "type": "structured", "expected_answer": expected_answer, + "evaluation_metadata": evaluation_metadata, "scoring_method": "static_json", } ) diff --git a/src/evaluation/models.py b/src/evaluation/models.py index 353619f23..7042f4c9e 100644 --- a/src/evaluation/models.py +++ b/src/evaluation/models.py @@ -23,6 +23,7 @@ class Scenario(BaseModel): category: str = "" characteristic_form: str | None = None expected_answer: str | None = None + evaluation_metadata: dict[str, Any] | None = None scoring_method: str | None = None @classmethod diff --git a/src/evaluation/report.py b/src/evaluation/report.py index 6738cfffe..7a80884b1 100644 --- a/src/evaluation/report.py +++ b/src/evaluation/report.py @@ -57,6 +57,8 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_key_match", "mode_exactly_one_key", "mode_term_coverage", + "mode_optional_term_coverage", + "car_score", ] score_values: dict[str, list[float]] = {name: [] for name in metric_names} @@ -127,6 +129,10 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_key_match_avg": _avg(score_values["mode_key_match"]), "mode_exactly_one_key_avg": _avg(score_values["mode_exactly_one_key"]), "mode_term_coverage_avg": _avg(score_values["mode_term_coverage"]), + "mode_optional_term_coverage_avg": _avg( + score_values["mode_optional_term_coverage"] + ), + "car_score_avg": _avg(score_values["car_score"]), "missing_keys_total": missing_keys_total, "extra_keys_total": extra_keys_total, "detail_entries_total": detail_entries_total, diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index fb719dc73..5b80a9708 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -78,6 +78,10 @@ class StaticJsonScore: mode_required_terms: list[str] = field(default_factory=list) mode_matched_terms: list[str] = field(default_factory=list) mode_term_coverage: float | None = None + mode_optional_terms: list[str] = field(default_factory=list) + mode_matched_optional_terms: list[str] = field(default_factory=list) + mode_optional_term_coverage: float | None = None + car_score: float | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable dictionary.""" @@ -125,56 +129,76 @@ def _strip_markdown_fence(content: str) -> str: return content -def _extract_balanced_structure(content: str) -> str: - """Extract first balanced {...}, [...], or (...) from noisy text.""" +def _balanced_structure_candidates( + content: str, + open_chars: set[str], +) -> list[str]: + """Return balanced structured substrings from noisy text. + + Parentheses are handled separately from JSON-like braces/brackets because + natural-language answers often contain explanatory parentheticals before + the final JSON object. + """ content = content.strip() + close_for_open = {"{": "}", "[": "]", "(": ")"} + candidates: list[str] = [] - candidates = [ - (content.find("{"), "{", "}"), - (content.find("["), "[", "]"), - (content.find("("), "(", ")"), - ] - candidates = [ - (idx, open_ch, close_ch) - for idx, open_ch, close_ch in candidates - if idx != -1 - ] + for start, open_ch in enumerate(content): + if open_ch not in open_chars: + continue - if not candidates: - return content + close_ch = close_for_open[open_ch] + depth = 0 + in_string = False + quote_char = "" + escaped = False + + for index in range(start, len(content)): + ch = content[index] + + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == quote_char: + in_string = False + continue - start, open_ch, close_ch = min(candidates, key=lambda item: item[0]) + if ch in {"'", '"'}: + in_string = True + quote_char = ch + continue - depth = 0 - in_string = False - quote_char = "" - escaped = False + if ch == open_ch: + depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + candidates.append(content[start : index + 1].strip()) + break - for index in range(start, len(content)): - ch = content[index] + return candidates - if in_string: - if escaped: - escaped = False - elif ch == "\\": - escaped = True - elif ch == quote_char: - in_string = False - continue - if ch in {"'", '"'}: - in_string = True - quote_char = ch - continue +def _parse_structured_candidate(content: str) -> tuple[bool, Any]: + """Try supported structured parsers for a candidate answer.""" + try: + return True, json.loads(content) + except json.JSONDecodeError: + pass - if ch == open_ch: - depth += 1 - elif ch == close_ch: - depth -= 1 - if depth == 0: - return content[start : index + 1].strip() + try: + return True, ast.literal_eval(content) + except (ValueError, SyntaxError): + pass - return content[start:].strip() + try: + return True, json.loads(content.replace("'", '"')) + except json.JSONDecodeError: + pass + + return False, None def _extract_count_from_text(content: str) -> int | float | None: @@ -268,22 +292,19 @@ def parse_structured_answer(value: Any) -> Any: content = extract_answer_text(value) content = _strip_markdown_fence(content) - content = _extract_balanced_structure(content) + parsed, result = _parse_structured_candidate(content) + if parsed: + return result - try: - return json.loads(content) - except json.JSONDecodeError: - pass + for candidate in _balanced_structure_candidates(content, {"{", "["}): + parsed, result = _parse_structured_candidate(candidate) + if parsed: + return result - try: - return ast.literal_eval(content) - except (ValueError, SyntaxError): - pass - - try: - return json.loads(content.replace("'", '"')) - except json.JSONDecodeError: - pass + for candidate in _balanced_structure_candidates(content, {"("}): + parsed, result = _parse_structured_candidate(candidate) + if parsed: + return result count = _extract_count_from_text(content) if count is not None: @@ -494,12 +515,44 @@ def _is_mode_gold_answer(value: Any) -> bool: return key in _MODE_KEYS -def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: - gold = parse_structured_answer(gold_answer) - model = parse_structured_answer(model_answer) +def _as_terms(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + value = [value] + if not isinstance(value, list): + return [] + return _dedupe_terms([str(item) for item in value]) + +def _mode_metadata_from_gold(gold_answer: Any) -> dict[str, Any]: + gold = parse_structured_answer(gold_answer) gold_key = str(next(iter(gold))).strip().lower() gold_value = next(iter(gold.values())) + return { + "mode": gold_key, + "required_terms": _extract_required_mode_terms(gold_value), + "optional_terms": [], + "must_have_exactly_one_mode_key": True, + } + + +def _evaluate_mode_json( + gold_answer: Any, + model_answer: Any, + evaluation_metadata: dict[str, Any] | None = None, +) -> StaticJsonScore: + gold = parse_structured_answer(gold_answer) + model = parse_structured_answer(model_answer) + + metadata = _mode_metadata_from_gold(gold_answer) + if evaluation_metadata: + metadata.update(evaluation_metadata) + + gold_key = str(metadata.get("mode") or next(iter(gold))).strip().lower() + required_terms = _as_terms(metadata.get("required_terms")) + optional_terms = _as_terms(metadata.get("optional_terms")) + must_have_one_key = bool(metadata.get("must_have_exactly_one_mode_key", True)) model_is_dict = isinstance(model, dict) model_keys = [str(key).strip().lower() for key in model.keys()] if model_is_dict else [] @@ -508,7 +561,6 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - required_terms = _extract_required_mode_terms(gold_value) model_text = _normalize_text_for_terms(model_value) matched_terms = [ term for term in required_terms if f" {term} " in f" {model_text} " @@ -516,6 +568,12 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: term_coverage = ( len(matched_terms) / len(required_terms) if required_terms else 1.0 ) + matched_optional_terms = [ + term for term in optional_terms if f" {term} " in f" {model_text} " + ] + optional_term_coverage = ( + len(matched_optional_terms) / len(optional_terms) if optional_terms else None + ) details = [ KeyComparison( @@ -543,13 +601,27 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: ) ) + for term in optional_terms: + matched = term in matched_optional_terms + details.append( + KeyComparison( + key=f"answer.optional_term.{term}", + gold_value=term, + model_value=term if matched else "MISSING", + exact=matched, + match_type="optional_term_present" if matched else "optional_term_missing", + similarity=1.0 if matched else 0.0, + accepted=matched, + ) + ) + missing_keys = [] if key_match else [f"answer.{gold_key}"] extra_keys = [] if model_is_dict: extra_keys = [ f"answer.{key}" for key in model_keys - if key not in {gold_key} or not model_exactly_one_key + if key not in {gold_key} or (must_have_one_key and not model_exactly_one_key) ] else: extra_keys = ["answer"] if model is not None else [] @@ -557,6 +629,10 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: total_gold_keys = 1 + len(required_terms) total_model_keys = 1 + len(required_terms) + len(extra_keys) exact_matches = (1 if key_match else 0) + len(matched_terms) + required_details = details[:total_gold_keys] + required_term_score = term_coverage + required_term_pass = bool(matched_terms) if required_terms else True + car_score = (0.6 * (1.0 if key_match else 0.0)) + (0.4 * required_term_score) precision = exact_matches / total_model_keys if total_model_keys else 0.0 recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 @@ -565,13 +641,19 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: if precision + recall > 0 else 0.0 ) - strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 + strict_exact = ( + 1.0 + if key_match + and required_term_pass + and (model_exactly_one_key or not must_have_one_key) + else 0.0 + ) return StaticJsonScore( partial_match_accuracy=recall, partial_exact_match_accuracy=recall, strict_exact_match_accuracy=strict_exact, - partial_similarity_score=sum(item.similarity for item in details) + partial_similarity_score=sum(item.similarity for item in required_details) / total_gold_keys, partial_numeric_match_accuracy=0.0, range_match_accuracy=0.0, @@ -598,6 +680,10 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: mode_required_terms=required_terms, mode_matched_terms=matched_terms, mode_term_coverage=term_coverage, + mode_optional_terms=optional_terms, + mode_matched_optional_terms=matched_optional_terms, + mode_optional_term_coverage=optional_term_coverage, + car_score=car_score, ) @@ -784,8 +870,12 @@ def evaluate_static_json( model_answer: Any, *, similarity_threshold: float = 0.0, + evaluation_metadata: dict[str, Any] | None = None, ) -> StaticJsonScore: """Evaluate one structured gold answer against one model answer.""" + if evaluation_metadata and str(evaluation_metadata.get("mode", "")).lower() in _MODE_KEYS: + return _evaluate_mode_json(gold_answer, model_answer, evaluation_metadata) + if _is_mode_gold_answer(gold_answer): return _evaluate_mode_json(gold_answer, model_answer) @@ -1013,13 +1103,20 @@ def __call__( ), ) - static_score = evaluate_static_json(gold_answer, answer) + static_score = evaluate_static_json( + gold_answer, + answer, + evaluation_metadata=scenario.evaluation_metadata, + ) passed = static_score.strict_exact_match_accuracy == 1.0 + score_value = static_score.car_score + if score_value is None: + score_value = static_score.f1 return ScorerResult( scorer=self.name, passed=passed, - score=round(static_score.f1, 3), + score=round(score_value, 3), rationale=( "strict structured match" if passed diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index c5de0049e..deef8a972 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -24,6 +24,29 @@ def test_parse_fenced_json_response_key_from_noisy_answer(): } +def test_parse_json_after_parenthetical_prose(): + raw = ( + 'The registry has generic descriptions (all descriptions are "Asset PMPxxxxx").\n\n' + '{"clarification": "Which pump do you mean by the big pump out the back?"}' + ) + + assert parse_structured_answer(raw) == { + "clarification": "Which pump do you mean by the big pump out the back?" + } + + +def test_parse_json_after_parenthetical_with_number(): + raw = ( + 'Based on the repair history (asset PMP42144 has 17 work orders), ' + 'the answer is:\n' + '{"clarification": "Which asset do you mean by the main unit?"}' + ) + + assert parse_structured_answer(raw) == { + "clarification": "Which asset do you mean by the main unit?" + } + + def test_parse_python_style_dict(): raw = "{'energy': 14, 'material': 48}" @@ -327,6 +350,97 @@ def test_mode_requires_exactly_one_top_level_key(): assert score.extra_keys == ["answer.clarification", "answer.response"] +def test_car_metadata_passes_on_mode_and_required_terms_only(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + {"clarification": "Which unit are you referring to as the main unit?"}, + evaluation_metadata={ + "mode": "clarification", + "required_terms": ["main unit"], + "optional_terms": ["asset"], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.car_score == 1.0 + assert score.mode_key_match == 1.0 + assert score.mode_required_terms == ["main unit"] + assert score.mode_matched_terms == ["main unit"] + assert score.mode_optional_terms == ["asset"] + assert score.mode_matched_optional_terms == [] + assert score.mode_optional_term_coverage == 0.0 + + +def test_car_optional_terms_do_not_inflate_similarity(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + { + "clarification": ( + "Which asset do you mean by the main unit? Please provide the " + "asset tag." + ) + }, + evaluation_metadata={ + "mode": "clarification", + "required_terms": ["main unit"], + "optional_terms": ["asset", "asset tag"], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.partial_similarity_score == 1.0 + assert score.mode_optional_term_coverage == 1.0 + + +def test_car_metadata_passes_when_any_required_term_matches(): + score = evaluate_static_json( + {"abstain": "Cannot determine because date or time is missing."}, + {"abstain": "Cannot determine this from the available date fields."}, + evaluation_metadata={ + "mode": "abstain", + "required_terms": ["lately", "date", "time"], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.car_score == 0.7333333333333333 + assert score.mode_key_match == 1.0 + assert score.mode_matched_terms == ["date"] + assert score.mode_term_coverage == 1 / 3 + + +def test_car_metadata_fails_when_required_term_is_missing(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + {"clarification": "Which asset should I analyze?"}, + evaluation_metadata={ + "mode": "clarification", + "required_terms": ["main unit"], + }, + ) + + assert score.strict_exact_match_accuracy == 0.0 + assert score.car_score == 0.6 + assert score.mode_key_match == 1.0 + assert score.mode_term_coverage == 0.0 + + +def test_car_metadata_fails_when_mode_key_is_wrong_even_with_required_term(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + {"response": "The main unit appears to be PMP42144."}, + evaluation_metadata={ + "mode": "clarification", + "required_terms": ["main unit"], + }, + ) + + assert score.strict_exact_match_accuracy == 0.0 + assert score.car_score == 0.4 + assert score.mode_key_match == 0.0 + assert score.mode_term_coverage == 1.0 + + def test_count_only_exact_match(): score = evaluate_static_json("34", "The answer is 34.") @@ -372,3 +486,29 @@ def test_static_json_scorer_wrapper_exact_match(): assert result.passed is True assert result.score == 1.0 assert result.details["strict_exact_match_accuracy"] == 1.0 + + +def test_static_json_scorer_uses_car_metadata_score(): + scenario = Scenario.from_raw( + { + "id": "151", + "text": "Clarify main unit.", + "expected_answer": '{"clarification": "Which asset do you mean by the main unit?"}', + "evaluation_metadata": { + "mode": "clarification", + "required_terms": ["main unit"], + }, + "scoring_method": "static_json", + } + ) + + scorer = StaticJsonScorer() + result = scorer( + scenario, + '{"clarification": "Which asset is the main unit?"}', + "", + ) + + assert result.passed is True + assert result.score == 1.0 + assert result.details["car_score"] == 1.0