From 095637f2a002bc02ff3fe3aec784d2640820bba7 Mon Sep 17 00:00:00 2001 From: Romain Chantereau Date: Mon, 27 Jul 2026 15:46:26 +0200 Subject: [PATCH] Fix TokenOverlap crash when references list is empty TokenOverlap.compute() raises ValueError when references is an empty list because max() is called on an empty generator. This occurs via the faithfulness metric when contexts is empty (e.g. vector store returns zero chunks due to eventual consistency). Add guard: if not results, return zero scores. Add test: test_token_overlap_empty_references. Co-Authored-By: Claude Signed-off-by: Romain Chantereau --- src/unitxt/metrics.py | 2 ++ tests/library/test_metrics.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/unitxt/metrics.py b/src/unitxt/metrics.py index 9942c1be6d..5e4930b255 100644 --- a/src/unitxt/metrics.py +++ b/src/unitxt/metrics.py @@ -4408,6 +4408,8 @@ def compute( self._compute_single_ref(str(reference), str(prediction)) for reference in references ] + if not results: + return {"precision": 0, "recall": 0, "f1": 0} return { measure: max(r[i] for r in results) for i, measure in enumerate(["precision", "recall", "f1"]) diff --git a/tests/library/test_metrics.py b/tests/library/test_metrics.py index 097856d267..d051273456 100644 --- a/tests/library/test_metrics.py +++ b/tests/library/test_metrics.py @@ -1190,6 +1190,17 @@ def test_token_overlap(self): for target, value in global_targets.items(): self.assertAlmostEqual(value, outputs[0]["score"]["global"][target]) + def test_token_overlap_empty_references(self): + metric = TokenOverlap() + predictions = ["hello there"] + references = [[]] + outputs = apply_metric( + metric=metric, predictions=predictions, references=references + ) + global_targets = {"f1": 0, "precision": 0, "recall": 0} + for target, value in global_targets.items(): + self.assertAlmostEqual(value, outputs[0]["score"]["global"][target]) + def test_roc_auc(self): metric = RocAuc() predictions = [0.2, 0.8, 1.0]