[BUG FIX] Normalize categorical answers in accuracy scoring

challenge/accuracy.py

@@ -0,0 +1,60 @@
+"""Utilities for robustly scoring categorical DriveLM answers."""
+
+import re
+from typing import Optional
+
+_EXPLICIT_CHOICE_RE = re.compile(
+    r"\b(?:answer|choice|option|selection|selected|correct(?:\s+answer)?|final)"
+    r"\b\s*(?:is|:|=|-)?\s*[([{]?\s*([A-Z])\b",
+    re.IGNORECASE,
+)
+_LEADING_CHOICE_RE = re.compile(
+    r"^\s*(?:[-*•]\s*)?[([{]?\s*([A-Z])\s*(?:[\])}.:,;-]|$)",
+    re.IGNORECASE,
+)
+_SINGLE_CHOICE_RE = re.compile(r"^[([{]?\s*([A-Z])\s*[\])}.:,;!?-]*$", re.IGNORECASE)
+
+
+def _normalize_text(value: object) -> str:
+    """Normalize case, whitespace, and terminal punctuation for exact matches."""
+    text = "" if value is None else str(value)
+    return re.sub(r"\s+", " ", text).strip().casefold().rstrip(".,;:!?)]}")
+
+
+def _single_choice(value: object) -> Optional[str]:
+    match = _SINGLE_CHOICE_RE.fullmatch(_normalize_text(value))
+    return match.group(1).upper() if match else None
+
+
+def extract_choice(answer: object) -> Optional[str]:
+    """Extract an explicitly selected option from a model response.
+
+    Explanations are accepted when they identify a choice (for example,
+    ``"The correct answer is A because ..."``).  A bare leading choice such as
+    ``"A. Going ahead"`` is also supported.  Arbitrary prose mentioning option
+    letters is intentionally rejected so that a sentence does not score by
+    accident.
+    """
+    text = "" if answer is None else str(answer)
+    explicit_matches = list(_EXPLICIT_CHOICE_RE.finditer(text))
+    if explicit_matches:
+        return explicit_matches[-1].group(1).upper()
+
+    leading_match = _LEADING_CHOICE_RE.match(text)
+    return leading_match.group(1).upper() if leading_match else None
+
+
+def answers_match(answer: object, ground_truth: object) -> bool:
+    """Return whether a prediction matches a categorical ground-truth answer.
+
+    Exact normalized matching remains the fallback for yes/no and free-form
+    labels.  When the ground truth is a single option letter, explanatory model
+    responses are compared by their selected option instead of their wording.
+    """
+    if _normalize_text(answer) == _normalize_text(ground_truth):
+        return True
+
+    expected_choice = _single_choice(ground_truth)
+    if expected_choice is None:
+        return False
+    return extract_choice(answer) == expected_choice

challenge/evaluation.py

@@ -8,6 +8,10 @@
 
 import sys
 sys.path.append(".")
+try:
+    from .accuracy import answers_match
+except ImportError:
+    from accuracy import answers_match
 from gpt_eval import GPTEvaluation
 
 
@@ -25,7 +29,7 @@ def eval_acc(self):
         for i in range(len(self.accuracy["answer"])):
             answer = self.accuracy["answer"][i]
             GT = self.accuracy["GT"][i]
-            if answer == GT:
+            if answers_match(answer, GT):
                 scores.append(1.0)
             else:
                 scores.append(0.0)

tests/test_accuracy.py

@@ -0,0 +1,30 @@
+from challenge.accuracy import answers_match, extract_choice
+
+
+def test_explanatory_multiple_choice_answer_matches_ground_truth():
+    answer = "The correct answer is A. The ego vehicle is steering to the left."
+
+    assert answers_match(answer, "A")
+
+
+def test_leading_option_label_matches_and_wrong_label_does_not():
+    assert answers_match("(B) Going straight at normal speed", "B")
+    assert not answers_match("The correct answer is B", "A")
+
+
+def test_option_letter_in_unrelated_prose_is_not_selected():
+    answer = "There is a sedan to the front; the options A and B describe motion."
+
+    assert extract_choice(answer) is None
+    assert not answers_match(answer, "A")
+
+
+def test_yes_no_and_exact_labels_keep_normalized_matching():
+    assert answers_match("No", "No.")
+    assert answers_match("  stopped  ", "Stopped")
+    assert not answers_match("Yes", "No.")
+
+
+def test_empty_or_non_choice_ground_truth_does_not_match_by_substring():
+    assert not answers_match("A", "")
+    assert not answers_match("The answer is A", "stopped")