feat(gooddata-eval): add KDA-skill agentic evaluator

packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py

@@ -30,6 +30,14 @@
     evaluate_agentic_guardrail,
     run_agentic_guardrail,
 )
+from gooddata_eval.core.agentic.kda_skill import (
+    AgenticKdaSummary,
+    KdaEvaluation,
+    KdaRunResult,
+    KdaSkillAssertionError,
+    evaluate_agentic_kda_skill,
+    run_agentic_kda_skill,
+)
 from gooddata_eval.core.agentic.metric_skill import (
     AgenticMetricSummary,
     MetricRunResult,
@@ -56,6 +64,7 @@
     "AgenticAlertSummary",
     "AgenticGeneralQuestionSummary",
     "AgenticGuardrailSummary",
+    "AgenticKdaSummary",
     "AgenticMetricSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
@@ -69,6 +78,9 @@
     "GeneralQuestionResult",
     "GuardrailAssertionError",
     "GuardrailResult",
+    "KdaEvaluation",
+    "KdaRunResult",
+    "KdaSkillAssertionError",
     "MetricRunResult",
     "MetricSkillAssertionError",
     "RunResult",
@@ -81,13 +93,15 @@
     "evaluate_agentic_conversation",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
+    "evaluate_agentic_kda_skill",
     "evaluate_agentic_metric_skill",
     "evaluate_agentic_search_tool",
     "evaluate_agentic_visualization",
     "run_agentic_alert_skill",
     "run_agentic_conversation",
     "run_agentic_general_question",
     "run_agentic_guardrail",
+    "run_agentic_kda_skill",
     "run_agentic_metric_skill",
     "run_agentic_search_tool",
     "run_agentic_visualization",

packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py

@@ -0,0 +1,25 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Shared "is this a clarifying question?" heuristic for agentic skill runners."""
+
+from __future__ import annotations
+
+
+def is_asking_clarification(text: str) -> bool:
+    """True if ``text`` reads as the agent asking the user for input, not a final answer.
+
+    Only the bare ``"?"``-anywhere check is tightened to require the message actually END
+    on a question -- a "?" anywhere in the text also matches a final answer that merely
+    quotes or rhetorically references a question, which would wrongly keep a single-turn
+    case going into a simulated-reply retry and could mask a real turn-1 failure behind an
+    artificial turn-2 pass. The other phrase checks stay substring-anywhere as before: they
+    weren't the source of that false-positive, and conversation.py's multi-turn, multi-skill
+    driver (up to 20 clarification rounds, not just KDA's single-turn case) relies on their
+    broader recall -- narrowing them too would risk the opposite failure, a real
+    disambiguation message going undetected and being graded as if it were the final answer.
+    """
+    if not text:
+        return False
+    t = text.strip().lower()
+    if t.endswith("?"):
+        return True
+    return "could you" in t or "please" in t or "clarif" in t

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -31,7 +31,12 @@ def __init__(self, raw: dict) -> None:
         self.id: str = raw.get("id", "")
         self.metadata: dict = raw.get("metadata") or {}
         self.session_id: str | None = raw.get("sessionId") or raw.get("session_id")
-        self.latency: float = float(raw.get("latency") or 0.0)
+        # None (missing/null) is preserved, not coerced to 0.0 -- a trace that hasn't
+        # finished ingesting has UNKNOWN latency, not zero latency, and callers (e.g.
+        # log_quality_and_value_scores below, and every skill's own `pt.latency if pt
+        # else None` gating) rely on that distinction to not treat "unknown" as the best
+        # possible outcome.
+        self.latency: float | None = float(raw["latency"]) if raw.get("latency") is not None else None
         self.total_cost: float = float(raw.get("totalCost") or raw.get("total_cost") or 0.0)
 
 
@@ -358,11 +363,24 @@ def log_quality_and_value_scores(
         data_type="NUMERIC",
         comment=f"{passed}/{total} strict checks passed",
     )
-    speed = 0.0 if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC)
-    cost_factor = 0.0 if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD)
-    value = _QUALITY_WEIGHT * quality + _SPEED_WEIGHT * speed + _COST_WEIGHT * cost_factor
+    # An unresolved latency/cost (trace not yet settled, price not available) is UNKNOWN,
+    # not the best (1.0) or worst (0.0) possible outcome -- substituting either would
+    # silently pull value_score toward one extreme. Drop that weighted term instead and
+    # renormalize over whichever components do have a real value, so value_score always
+    # reflects only the signals actually measured for this run.
+    components = [(_QUALITY_WEIGHT, quality)]
+    speed = None if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC)
+    if speed is not None:
+        components.append((_SPEED_WEIGHT, speed))
+    cost_factor = None if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD)
+    if cost_factor is not None:
+        components.append((_COST_WEIGHT, cost_factor))
+    weight_total = sum(w for w, _ in components)
+    value = sum(w * v for w, v in components) / weight_total
     latency_str = "unknown" if latency_sec is None else f"{latency_sec:.2f}s"
     cost_str = "unknown" if cost_usd is None else f"${cost_usd:.4f}"
+    speed_str = "n/a" if speed is None else f"{speed:.2f}"
+    cost_factor_str = "n/a" if cost_factor is None else f"{cost_factor:.2f}"
     score_safe(
         langfuse,
         trace_id,
@@ -371,8 +389,8 @@ def log_quality_and_value_scores(
         data_type="NUMERIC",
         comment=(
             f"{_QUALITY_WEIGHT}*quality({quality:.2f}) + "
-            f"{_SPEED_WEIGHT}*speed({speed:.2f}) + "
-            f"{_COST_WEIGHT}*cost({cost_factor:.2f}); "
+            f"{_SPEED_WEIGHT}*speed({speed_str}) + "
+            f"{_COST_WEIGHT}*cost({cost_factor_str}), renormalized /{weight_total:.1f}; "
             f"latency={latency_str}; cost={cost_str}"
         ),
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -11,6 +11,7 @@
 from gooddata_sdk import GoodDataSdk
 from pydantic import BaseModel
 
+from gooddata_eval.core.agentic._clarification import is_asking_clarification
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
@@ -192,13 +193,6 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
     return None
 
 
-def _is_asking_clarification(text: str) -> bool:
-    if not text:
-        return False
-    t = text.lower()
-    return "?" in t or "could you" in t or "please" in t or "clarif" in t
-
-
 def _get_sim_user_response(agent_message: str, turn: TurnDefinition, expected_output: dict | None) -> str:
     """Generate a simulated user reply to an agent clarification question."""
     otype = turn.expected_output_type
@@ -327,7 +321,7 @@ def run_agentic_conversation(
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
-                asking = _is_asking_clarification(response_text) or bool(chat_result.alert_proposals)
+                asking = is_asking_clarification(response_text) or bool(chat_result.alert_proposals)
                 if asking and clarification_turns < max_clarification_turns:
                     clarification_turns += 1
                     total_clarification_turns += 1