From e7f95327321f00d0f6f9acf0ca1c3c9e54d2de7a Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 16 Jul 2026 13:57:17 -0700 Subject: [PATCH 1/3] fix(eval): clamp out-of-range scores from LLM judge tool calls gemini-2.5-flash occasionally returns corrupted numeric scores in its submit_evaluation tool call despite the 0-100 range stated in the tool schema (observed in production: score=989898 with a justification saying the outputs "match perfectly", and score=950 where 95 was intended). extract_tool_call_response passed these values through unvalidated, and no downstream layer on the serverless eval path clamps them, so a single corrupted item blew a 64-item run-level average up to 15559.13%. Clamp finite out-of-range scores to 0-100, prepend a visible warning to the justification so the correction is auditable in the UI, and reject non-finite scores with a ValueError. Co-Authored-By: Claude Fable 5 --- .../eval/evaluators/legacy_llm_helpers.py | 23 ++++++ .../evaluators/test_legacy_llm_helpers.py | 81 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 packages/uipath/tests/evaluators/test_legacy_llm_helpers.py diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py index 67aaee0c3..c19152f55 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py @@ -1,6 +1,7 @@ """Helper functions for legacy LLM evaluators using function calling.""" import logging +import math from typing import Any from uipath.platform.chat.llm_gateway import ( @@ -92,6 +93,28 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse: score = float(arguments["score"]) justification = str(arguments["justification"]) + if not math.isfinite(score): + error_msg = ( + f"Non-finite score {score!r} in tool call arguments from model {model}" + ) + logger.error(f"❌ {error_msg}") + raise ValueError(error_msg) + + # Models occasionally emit corrupted numeric tool arguments despite the + # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning + # 989898 or 950). An unclamped value poisons every run-level aggregate + # downstream, so clamp here and surface the correction in the justification. + if score < 0.0 or score > 100.0: + clamped = max(0.0, min(100.0, score)) + logger.warning( + f"⚠️ Model {model} returned out-of-range score {score}; clamping to {clamped}" + ) + justification = ( + f"[Warning: model returned out-of-range score {score}; " + f"clamped to {clamped}.]\n{justification}" + ) + score = clamped + logger.debug( f"✅ Extracted score: {score}, justification length: {len(justification)} chars" ) diff --git a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py new file mode 100644 index 000000000..27e254ecb --- /dev/null +++ b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py @@ -0,0 +1,81 @@ +"""Tests for legacy LLM helper functions (submit_evaluation tool-call parsing).""" + +from types import SimpleNamespace +from typing import Any + +import pytest + +from uipath.eval.evaluators.legacy_llm_helpers import extract_tool_call_response + + +def _make_response(arguments: dict[str, Any]) -> Any: + """Build a minimal chat-completions response carrying a submit_evaluation tool call.""" + tool_call = SimpleNamespace(arguments=arguments) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class TestExtractToolCallResponse: + """Test extract_tool_call_response score validation and clamping.""" + + def test_valid_score_passes_through(self) -> None: + response = _make_response({"score": 88, "justification": "ok"}) + + result = extract_tool_call_response(response, "gemini-2.5-flash") + + assert result.score == 88.0 + assert result.justification == "ok" + + def test_out_of_range_score_is_clamped_to_100(self) -> None: + # Real payload observed in production: gemini-2.5-flash returned + # score=989898 in its submit_evaluation tool call while the justification + # said the outputs "match perfectly". Unclamped, this single value blew a + # 64-item run-level average up to 15559.13%. + response = _make_response( + {"score": 989898, "justification": "matches perfectly"} + ) + + result = extract_tool_call_response(response, "gemini-2.5-flash") + + assert result.score == 100.0 + assert "out-of-range score 989898" in result.justification + assert "matches perfectly" in result.justification + + def test_out_of_range_950_is_clamped(self) -> None: + # Second production occurrence from the same eval run: score=950 + # (the model most likely intended 95). + response = _make_response({"score": 950, "justification": "equivalent"}) + + result = extract_tool_call_response(response, "gemini-2.5-flash") + + assert result.score == 100.0 + assert "out-of-range score 950" in result.justification + + def test_negative_score_is_clamped_to_0(self) -> None: + response = _make_response({"score": -5, "justification": "bad"}) + + result = extract_tool_call_response(response, "gpt-4o") + + assert result.score == 0.0 + + @pytest.mark.parametrize("boundary", [0, 100]) + def test_boundary_scores_not_modified(self, boundary: int) -> None: + response = _make_response({"score": boundary, "justification": "j"}) + + result = extract_tool_call_response(response, "m") + + assert result.score == float(boundary) + assert result.justification == "j" + + def test_non_finite_score_raises(self) -> None: + response = _make_response({"score": float("nan"), "justification": "j"}) + + with pytest.raises(ValueError, match="Non-finite score"): + extract_tool_call_response(response, "m") + + def test_missing_score_raises(self) -> None: + response = _make_response({"justification": "j"}) + + with pytest.raises(ValueError, match="Missing 'score'"): + extract_tool_call_response(response, "m") From e3edf26efe80b0980aca48ae340551fc1a8e69a0 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 16 Jul 2026 15:39:58 -0700 Subject: [PATCH 2/3] fix(eval): reject out-of-range judge scores instead of clamping Review feedback: clamping fabricates a score the model never gave (950 clamped to 100 when the model likely meant 95). Reject invalid scores with a ValueError instead; track_evaluation_metrics converts it into an ErrorEvaluationResult so the eval item surfaces as an evaluator error with the invalid value in the details. Co-Authored-By: Claude Fable 5 --- .../eval/evaluators/legacy_llm_helpers.py | 25 ++++-------- .../evaluators/test_legacy_llm_helpers.py | 40 +++++++++---------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py index c19152f55..08d3be345 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py @@ -93,28 +93,19 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse: score = float(arguments["score"]) justification = str(arguments["justification"]) - if not math.isfinite(score): + # Models occasionally emit corrupted numeric tool arguments despite the + # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning + # 989898 or 950). Unvalidated, such a value poisons every run-level + # aggregate downstream, so reject it and let the evaluation surface as + # an error instead of recording a fabricated score. + if not math.isfinite(score) or score < 0.0 or score > 100.0: error_msg = ( - f"Non-finite score {score!r} in tool call arguments from model {model}" + f"Invalid score {score!r} in tool call arguments from model {model}: " + f"expected a number between 0 and 100" ) logger.error(f"❌ {error_msg}") raise ValueError(error_msg) - # Models occasionally emit corrupted numeric tool arguments despite the - # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning - # 989898 or 950). An unclamped value poisons every run-level aggregate - # downstream, so clamp here and surface the correction in the justification. - if score < 0.0 or score > 100.0: - clamped = max(0.0, min(100.0, score)) - logger.warning( - f"⚠️ Model {model} returned out-of-range score {score}; clamping to {clamped}" - ) - justification = ( - f"[Warning: model returned out-of-range score {score}; " - f"clamped to {clamped}.]\n{justification}" - ) - score = clamped - logger.debug( f"✅ Extracted score: {score}, justification length: {len(justification)} chars" ) diff --git a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py index 27e254ecb..94092a3ef 100644 --- a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py +++ b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py @@ -17,7 +17,7 @@ def _make_response(arguments: dict[str, Any]) -> Any: class TestExtractToolCallResponse: - """Test extract_tool_call_response score validation and clamping.""" + """Test extract_tool_call_response score validation.""" def test_valid_score_passes_through(self) -> None: response = _make_response({"score": 88, "justification": "ok"}) @@ -27,40 +27,35 @@ def test_valid_score_passes_through(self) -> None: assert result.score == 88.0 assert result.justification == "ok" - def test_out_of_range_score_is_clamped_to_100(self) -> None: + def test_out_of_range_score_is_rejected(self) -> None: # Real payload observed in production: gemini-2.5-flash returned # score=989898 in its submit_evaluation tool call while the justification - # said the outputs "match perfectly". Unclamped, this single value blew a - # 64-item run-level average up to 15559.13%. + # said the outputs "match perfectly". Unvalidated, this single value blew + # a 64-item run-level average up to 15559.13%. The evaluation must surface + # as an error rather than record a fabricated score. response = _make_response( {"score": 989898, "justification": "matches perfectly"} ) - result = extract_tool_call_response(response, "gemini-2.5-flash") - - assert result.score == 100.0 - assert "out-of-range score 989898" in result.justification - assert "matches perfectly" in result.justification + with pytest.raises(ValueError, match="Invalid score 989898"): + extract_tool_call_response(response, "gemini-2.5-flash") - def test_out_of_range_950_is_clamped(self) -> None: + def test_out_of_range_950_is_rejected(self) -> None: # Second production occurrence from the same eval run: score=950 # (the model most likely intended 95). response = _make_response({"score": 950, "justification": "equivalent"}) - result = extract_tool_call_response(response, "gemini-2.5-flash") + with pytest.raises(ValueError, match="Invalid score 950"): + extract_tool_call_response(response, "gemini-2.5-flash") - assert result.score == 100.0 - assert "out-of-range score 950" in result.justification - - def test_negative_score_is_clamped_to_0(self) -> None: + def test_negative_score_is_rejected(self) -> None: response = _make_response({"score": -5, "justification": "bad"}) - result = extract_tool_call_response(response, "gpt-4o") - - assert result.score == 0.0 + with pytest.raises(ValueError, match="Invalid score -5"): + extract_tool_call_response(response, "gpt-4o") @pytest.mark.parametrize("boundary", [0, 100]) - def test_boundary_scores_not_modified(self, boundary: int) -> None: + def test_boundary_scores_accepted(self, boundary: int) -> None: response = _make_response({"score": boundary, "justification": "j"}) result = extract_tool_call_response(response, "m") @@ -68,10 +63,11 @@ def test_boundary_scores_not_modified(self, boundary: int) -> None: assert result.score == float(boundary) assert result.justification == "j" - def test_non_finite_score_raises(self) -> None: - response = _make_response({"score": float("nan"), "justification": "j"}) + @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) + def test_non_finite_score_is_rejected(self, value: float) -> None: + response = _make_response({"score": value, "justification": "j"}) - with pytest.raises(ValueError, match="Non-finite score"): + with pytest.raises(ValueError, match="Invalid score"): extract_tool_call_response(response, "m") def test_missing_score_raises(self) -> None: From 69203b4dad3c51d67992427c05338fb3b3c2969f Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Wed, 22 Jul 2026 11:31:44 -0700 Subject: [PATCH 3/3] fix(eval): wrap score parsing errors with context, bump version to 2.13.14 - Wrap float(arguments["score"]) in try/except so non-numeric or null scores raise a contextual ValueError with structured logging instead of bubbling up raw (addresses review comment) - Bump uipath to 2.13.14 (2.13.13 already published on PyPI) Co-Authored-By: Claude Fable 5 --- packages/uipath/pyproject.toml | 2 +- .../src/uipath/eval/evaluators/legacy_llm_helpers.py | 12 +++++++++++- .../tests/evaluators/test_legacy_llm_helpers.py | 7 +++++++ packages/uipath/uv.lock | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 2e5404478..ec813ed17 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.13" +version = "2.13.14" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py index 08d3be345..8cb8da0de 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py @@ -90,7 +90,17 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse: logger.debug(f"Arguments: {arguments}") raise ValueError(error_msg) - score = float(arguments["score"]) + try: + score = float(arguments["score"]) + except (ValueError, TypeError) as e: + error_msg = ( + f"Non-numeric score {arguments['score']!r} in tool call arguments " + f"from model {model}: expected a number between 0 and 100" + ) + logger.error(f"❌ {error_msg}") + logger.debug(f"Arguments: {arguments}") + raise ValueError(error_msg) from e + justification = str(arguments["justification"]) # Models occasionally emit corrupted numeric tool arguments despite the diff --git a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py index 94092a3ef..c44f8d855 100644 --- a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py +++ b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py @@ -75,3 +75,10 @@ def test_missing_score_raises(self) -> None: with pytest.raises(ValueError, match="Missing 'score'"): extract_tool_call_response(response, "m") + + @pytest.mark.parametrize("value", ["not-a-number", None, [95]]) + def test_non_numeric_score_is_rejected(self, value: Any) -> None: + response = _make_response({"score": value, "justification": "j"}) + + with pytest.raises(ValueError, match="Non-numeric score"): + extract_tool_call_response(response, "m") diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 1314ef227..cca93a5f9 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.13" +version = "2.13.14" source = { editable = "." } dependencies = [ { name = "applicationinsights" },