From cc15d3085042763b1c97a0e5c2200b36c3173e27 Mon Sep 17 00:00:00 2001 From: Chathurangi Shyalika Date: Wed, 29 Jul 2026 15:42:56 -0400 Subject: [PATCH] Fix evaluator extraction for noisy final numeric answers --- src/evaluation/scorers/static_json.py | 124 +++++++++++++----- .../tests/test_static_json_scorer.py | 45 +++++++ 2 files changed, 134 insertions(+), 35 deletions(-) diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index fb719dc73..7afa9d564 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -125,26 +125,10 @@ def _strip_markdown_fence(content: str) -> str: return content -def _extract_balanced_structure(content: str) -> str: - """Extract first balanced {...}, [...], or (...) from noisy text.""" - content = content.strip() - - candidates = [ - (content.find("{"), "{", "}"), - (content.find("["), "[", "]"), - (content.find("("), "(", ")"), - ] - candidates = [ - (idx, open_ch, close_ch) - for idx, open_ch, close_ch in candidates - if idx != -1 - ] - - if not candidates: - return content - - start, open_ch, close_ch = min(candidates, key=lambda item: item[0]) - +def _balanced_from_index( + content: str, start: int, open_ch: str, close_ch: str +) -> str | None: + """Return the balanced structure starting at ``start``, if present.""" depth = 0 in_string = False quote_char = "" @@ -174,7 +158,55 @@ def _extract_balanced_structure(content: str) -> str: if depth == 0: return content[start : index + 1].strip() - return content[start:].strip() + return None + + +def _extract_balanced_structures(content: str) -> list[str]: + """Extract parseable-looking {...}, [...], and (...) candidates from noisy text.""" + content = content.strip() + candidates: list[tuple[int, int, str]] = [] + + for priority, (open_ch, close_ch) in enumerate( + [("{", "}"), ("[", "]"), ("(", ")")] + ): + start = content.find(open_ch) + while start != -1: + candidate = _balanced_from_index(content, start, open_ch, close_ch) + if candidate is not None: + candidates.append((priority, start, candidate)) + break + start = content.find(open_ch, start + 1) + + return [candidate for _, _, candidate in sorted(candidates)] + + +def _extract_balanced_structure(content: str) -> str: + """Extract the first balanced {...}, [...], or (...) candidate from noisy text.""" + candidates = _extract_balanced_structures(content) + return candidates[0] if candidates else content.strip() + + +_PARSE_MISSING = object() + + +def _parse_json_or_python(content: str) -> Any: + """Parse JSON/Python literal text, returning a sentinel on failure.""" + try: + return json.loads(content) + except json.JSONDecodeError: + pass + + try: + return ast.literal_eval(content) + except (ValueError, SyntaxError): + pass + + try: + return json.loads(content.replace("'", '"')) + except json.JSONDecodeError: + pass + + return _PARSE_MISSING def _extract_count_from_text(content: str) -> int | float | None: @@ -198,6 +230,32 @@ def _extract_count_from_text(content: str) -> int | float | None: return None +def _extract_final_count_from_text(content: str) -> int | float | None: + """Extract a final standalone count from a noisy scalar answer.""" + stripped = content.strip() + count = _extract_count_from_text(stripped) + if count is not None: + return count + + final_number = re.compile( + r"^\s*(?:" + r"(?:final\s+answer|answer|count|result)\s*(?:is|:)?\s*" + r")?(-?\d+(?:\.\d+)?)\s*\.?\s*$", + flags=re.IGNORECASE, + ) + for line in reversed(stripped.splitlines()): + line = line.strip() + if not line: + continue + match = final_number.fullmatch(line) + if match: + number = match.group(1) + return float(number) if "." in number else int(number) + break + + return None + + _CHOICE_LETTER_RE = re.compile(r"^[A-Za-z]$") _CHOICE_LINE_RE = re.compile( r"^\s*(?:" @@ -268,22 +326,18 @@ def parse_structured_answer(value: Any) -> Any: content = extract_answer_text(value) content = _strip_markdown_fence(content) - content = _extract_balanced_structure(content) - - try: - return json.loads(content) - except json.JSONDecodeError: - pass + parsed = _parse_json_or_python(content) + if parsed is not _PARSE_MISSING: + return parsed - try: - return ast.literal_eval(content) - except (ValueError, SyntaxError): - pass + for candidate in _extract_balanced_structures(content): + parsed = _parse_json_or_python(candidate) + if parsed is not _PARSE_MISSING: + return parsed - try: - return json.loads(content.replace("'", '"')) - except json.JSONDecodeError: - pass + count = _extract_final_count_from_text(content) + if count is not None: + return count count = _extract_count_from_text(content) if count is not None: diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index c5de0049e..94a2e094b 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -47,10 +47,55 @@ def test_parse_noisy_count_answer(): assert parse_structured_answer("The answer is 34.") == 34 +def test_parse_noisy_count_answer_prefers_final_standalone_number(): + raw = ( + "I checked the work orders and found no normal-operation jobs " + "(such as tramming or normal machine movement) that should be counted.\n\n" + "0" + ) + + assert parse_structured_answer(raw) == 0 + + +def test_count_answer_compares_final_number_not_parenthetical_text(): + score = evaluate_static_json( + "0", + ( + "I checked the work orders and found no normal-operation jobs " + "(such as tramming or normal machine movement) that should be counted.\n\n" + "0" + ), + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.f1 == 1.0 + assert score.details[0].model_value == "0" + + +def test_count_answer_with_wrong_final_number_fails_against_final_number(): + score = evaluate_static_json( + "52", + ( + 'One work order mentions both "engine" and "motor"; this concerns ' + "an engine (the engine fan), so it counts as an engine job.\n\n" + "55" + ), + ) + + assert score.strict_exact_match_accuracy == 0.0 + assert score.details[0].model_value == "55" + + def test_parse_fault_code_as_categorical_string(): assert parse_structured_answer("FC101") == "FC101" +def test_parse_json_object_after_parenthetical_text(): + raw = 'The dataset is ambiguous (no asset is specified).\n\n{"clarification": "Which asset?"}' + + assert parse_structured_answer(raw) == {"clarification": "Which asset?"} + + def test_choice_answer_accepts_final_standalone_letter_after_explanation(): score = evaluate_static_json( "C",