feat(eval): dataset-level evaluators + classification schemas, samples, e2e

packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py

@@ -0,0 +1,62 @@
+"""Aggregator specs embedded in per-datapoint classification evaluator configs.
+
+Each aggregator is a run-level metric (precision / recall / f-score) attached
+to a classification evaluator. Classes are declared once on the parent evaluator
+config — every aggregator on the same evaluator operates on the same class
+vocabulary, so the field is not repeated per spec. Only the metric-shape fields
+(``averaging`` and, for fscore, ``f_value``) live on the spec itself.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+
+class _AggregatorSpecBase(BaseModel):
+    """Shared pydantic config for every aggregator variant."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+
+class PrecisionAggregatorSpec(_AggregatorSpecBase):
+    """Run-level precision aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["precision"] = "precision"
+    averaging: Literal["macro", "micro"]
+
+
+class RecallAggregatorSpec(_AggregatorSpecBase):
+    """Run-level recall aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["recall"] = "recall"
+    averaging: Literal["macro", "micro"]
+
+
+class FScoreAggregatorSpec(_AggregatorSpecBase):
+    """Run-level F-beta aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["fscore"] = "fscore"
+    averaging: Literal["macro", "micro"]
+    # Upper bound keeps beta² finite — a huge beta overflows to inf and the
+    # F-score becomes NaN, which is not representable in JSON.
+    f_value: float = Field(default=1.0, gt=0, le=1000)
+
+
+class ConfusionMatrixAggregatorSpec(_AggregatorSpecBase):
+    """Run-level raw k×k confusion matrix — no scalar headline, no averaging."""
+
+    type: Literal["confusion_matrix"] = "confusion_matrix"
+
+
+AggregatorSpec = Annotated[
+    Union[
+        PrecisionAggregatorSpec,
+        RecallAggregatorSpec,
+        FScoreAggregatorSpec,
+        ConfusionMatrixAggregatorSpec,
+    ],
+    Field(discriminator="type"),
+]

packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py

@@ -0,0 +1,45 @@
+"""Base abstractions for dataset-level evaluators.
+
+A dataset-level evaluator runs once per evaluation set, after all per-datapoint
+evaluators have produced their results. It consumes the per-datapoint
+EvaluationResultDto values from one named source evaluator and emits a single
+EvaluationResult that summarizes the dataset.
+
+Concretely distinct from GenericBaseEvaluator: different evaluate() signature,
+different lifecycle. Kept as a parallel hierarchy rather than a subclass so the
+runtime cannot accidentally dispatch a dataset evaluator through the
+per-datapoint loop.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from ..models.models import EvaluationResult, EvaluationResultDto
+from ._aggregator_specs import AggregatorSpec
+
+
+class BaseDatasetEvaluator(ABC):
+    """Abstract base for dataset-level evaluators.
+
+    Constructed from an :class:`AggregatorSpec`, the source evaluator's name,
+    and the class vocabulary of the parent per-datapoint evaluator. Classes
+    live on the evaluator config (not the spec) — every aggregator on the same
+    evaluator operates on the same vocabulary.
+    """
+
+    spec: AggregatorSpec
+    source_evaluator: str
+    classes: list[str]
+
+    def __init__(
+        self, spec: AggregatorSpec, source_evaluator: str, classes: list[str]
+    ) -> None:
+        """Store the aggregator spec, source evaluator name, and shared classes."""
+        self.spec = spec
+        self.source_evaluator = source_evaluator
+        self.classes = classes
+
+    @abstractmethod
+    def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult:
+        """Reduce per-datapoint results into a single run-level EvaluationResult."""

packages/uipath/src/uipath/eval/evaluators/base_evaluator.py

@@ -47,6 +47,22 @@ class BaseEvaluatorJustification(BaseModel):
     expected: str
     actual: str
 
+    @classmethod
+    def try_from(cls, details: object) -> "BaseEvaluatorJustification | None":
+        """Coerce a free-form details payload into a justification, or return None.
+
+        Accepts either an existing instance or a dict that ``model_validate`` can
+        parse. Anything else (str, None, malformed dict) yields ``None``.
+        """
+        if isinstance(details, cls):
+            return details
+        if isinstance(details, dict):
+            try:
+                return cls.model_validate(details)
+            except Exception:
+                return None
+        return None
+
 
 # Additional type variables for Config and Justification
 # Note: C must be BaseEvaluatorConfig[T] to ensure type consistency

packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py

@@ -0,0 +1,261 @@
+"""Dataset-level classification evaluators: Precision, Recall, F-score, Confusion Matrix.
+
+All variants share the same internal machinery — a k x k confusion matrix built
+from each per-datapoint result's BaseEvaluatorJustification (expected, actual)
+strings. The scalar variants (precision / recall / fscore) emit per-class
+metrics plus micro/macro averages and pick the headline ``score`` per the
+spec's ``averaging``; the ``confusion_matrix`` variant emits only the raw grid
+with a 0.0 placeholder score.
+
+The ``details`` payload is the platform wire contract: the Agents reducer
+worker (python-dataset-eval-worker) calls this evaluator and ships
+``details.model_dump(by_alias=True, exclude_none=True)`` verbatim to the C#
+backend, where the frontend's zod schema
+(frontend-sw/src/schemas/evaluations/evals.ts) validates it. Changing field
+names or shapes here is a cross-repo breaking change.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+from ..models.models import (
+    EvaluationResult,
+    EvaluationResultDto,
+    NumericEvaluationResult,
+)
+from ._aggregator_specs import (
+    ConfusionMatrixAggregatorSpec,
+    FScoreAggregatorSpec,
+)
+from .base_dataset_evaluator import BaseDatasetEvaluator
+from .base_evaluator import BaseEvaluatorJustification
+
+
+class PerClassMetrics(BaseModel):
+    """Per-class confusion counts plus all three scalar metrics for that class."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    tp: int
+    tn: int
+    fp: int
+    fn: int
+    support: int
+    precision: float
+    recall: float
+    f_score: float
+
+
+class AveragedMetrics(BaseModel):
+    """Micro- or macro-averaged precision / recall / F-score triple."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    precision: float
+    recall: float
+    f_score: float
+
+
+class ClassificationDetails(BaseModel):
+    """Structured details payload emitted by every classification aggregator.
+
+    The scalar metrics (precision / recall / fscore) populate every field;
+    the ``confusion_matrix`` variant emits only the grid + counts, leaving
+    ``averaging`` / ``f_value`` / ``per_class`` / ``macro`` / ``micro`` as
+    None (excluded from the wire dump).
+    """
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    metric: str
+    classes: list[str]
+    confusion_matrix: list[list[int]] = Field(
+        ...,
+        description=(
+            "k x k confusion matrix indexed as "
+            "``confusion_matrix[predicted_idx][expected_idx]`` "
+            "(rows are predicted classes, columns are expected). "
+            "This is the transpose of sklearn's convention "
+            "(``[true][predicted]``); UI / consumer code must use the "
+            "orientation documented here."
+        ),
+    )
+    n_total: int
+    n_scored: int
+    n_skipped: int
+    averaging: str | None = None
+    f_value: float | None = None
+    per_class: dict[str, PerClassMetrics] | None = None
+    macro: AveragedMetrics | None = None
+    micro: AveragedMetrics | None = None
+
+
+@dataclass(slots=True)
+class _ConfusionData:
+    """Internal: confusion matrix and per-class counts derived from results."""
+
+    classes: list[str]
+    matrix: list[list[int]]
+    n_total: int
+    n_scored: int
+    n_skipped: int
+
+
+def _build_confusion(
+    results: list[EvaluationResultDto],
+    classes: list[str],
+) -> _ConfusionData:
+    """Build a confusion matrix from per-datapoint results.
+
+    Results without a parseable justification are counted in ``n_skipped`` and
+    omitted from the matrix. Pairs whose expected or actual label isn't in
+    ``classes`` are also skipped. Labels are normalized to lowercase for the
+    lookup index so a classifier returning "Book" vs configured "book" still
+    matches, but the user-supplied casing is preserved in the returned
+    ``_ConfusionData.classes`` so downstream output (per_class keys, UI labels)
+    shows what the user typed.
+    """
+    index_of = {c.lower(): i for i, c in enumerate(classes)}
+    k = len(classes)
+    matrix = [[0] * k for _ in range(k)]
+
+    n_total = len(results)
+    n_scored = 0
+    n_skipped = 0
+
+    for r in results:
+        j = BaseEvaluatorJustification.try_from(r.details)
+        if j is None:
+            n_skipped += 1
+            continue
+        exp = j.expected.lower()
+        act = j.actual.lower()
+        if exp not in index_of or act not in index_of:
+            n_skipped += 1
+            continue
+        matrix[index_of[act]][index_of[exp]] += 1
+        n_scored += 1
+
+    return _ConfusionData(
+        classes=list(classes),
+        matrix=matrix,
+        n_total=n_total,
+        n_scored=n_scored,
+        n_skipped=n_skipped,
+    )
+
+
+def _f_beta(precision: float, recall: float, beta: float) -> float:
+    b2 = beta * beta
+    # denom == 0 iff precision == recall == 0 (both terms are non-negative and
+    # beta > 0), which is exactly the zero-score case.
+    denom = b2 * precision + recall
+    if denom == 0:
+        return 0.0
+    return (1 + b2) * precision * recall / denom
+
+
+class ClassificationDatasetEvaluator(BaseDatasetEvaluator):
+    """One implementation for all classification aggregators.
+
+    Scalar variants (precision / recall / fscore) compute the full per-class
+    P/R/F report and pick the headline by the spec's ``averaging``; the
+    ``confusion_matrix`` variant returns only the raw grid.
+    """
+
+    def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult:
+        """Compute the configured metric report and return the headline as score."""
+        confusion = _build_confusion(results, self.classes)
+
+        if isinstance(self.spec, ConfusionMatrixAggregatorSpec):
+            # No scalar headline — emit the raw grid and let the UI render it.
+            details = ClassificationDetails(
+                metric=self.spec.type,
+                classes=confusion.classes,
+                confusion_matrix=confusion.matrix,
+                n_total=confusion.n_total,
+                n_scored=confusion.n_scored,
+                n_skipped=confusion.n_skipped,
+            )
+            return NumericEvaluationResult(score=0.0, details=details)
+
+        f_value = (
+            self.spec.f_value if isinstance(self.spec, FScoreAggregatorSpec) else 1.0
+        )
+        k = len(confusion.classes)
+
+        per_class: dict[str, PerClassMetrics] = {}
+        precisions: list[float] = []
+        recalls: list[float] = []
+        f_scores: list[float] = []
+        total_tp = total_fp = total_fn = 0
+
+        for c, label in enumerate(confusion.classes):
+            tp = confusion.matrix[c][c]
+            row_sum = sum(confusion.matrix[c])  # predicted as `label`
+            col_sum = sum(confusion.matrix[j][c] for j in range(k))  # true `label`
+            fp = row_sum - tp
+            fn = col_sum - tp
+            tn = confusion.n_scored - tp - fp - fn
+
+            precision = tp / row_sum if row_sum > 0 else 0.0
+            recall = tp / col_sum if col_sum > 0 else 0.0
+            f_score = _f_beta(precision, recall, f_value)
+
+            per_class[label] = PerClassMetrics(
+                tp=tp,
+                tn=tn,
+                fp=fp,
+                fn=fn,
+                support=tp + fn,
+                precision=precision,
+                recall=recall,
+                f_score=f_score,
+            )
+            precisions.append(precision)
+            recalls.append(recall)
+            f_scores.append(f_score)
+            total_tp += tp
+            total_fp += fp
+            total_fn += fn
+
+        # AggregatorSpec classes come from the ExactMatch config which requires
+        # a non-empty list, so k >= 1 always.
+        macro = AveragedMetrics(
+            precision=sum(precisions) / k,
+            recall=sum(recalls) / k,
+            f_score=sum(f_scores) / k,
+        )
+        micro_p = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0.0
+        micro_r = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0.0
+        micro = AveragedMetrics(
+            precision=micro_p,
+            recall=micro_r,
+            f_score=_f_beta(micro_p, micro_r, f_value),
+        )
+
+        averaged = micro if self.spec.averaging == "micro" else macro
+        headline = {
+            "precision": averaged.precision,
+            "recall": averaged.recall,
+            "fscore": averaged.f_score,
+        }[self.spec.type]
+
+        details = ClassificationDetails(
+            metric=self.spec.type,
+            averaging=self.spec.averaging,
+            f_value=f_value,
+            classes=confusion.classes,
+            confusion_matrix=confusion.matrix,
+            per_class=per_class,
+            macro=macro,
+            micro=micro,
+            n_total=confusion.n_total,
+            n_scored=confusion.n_scored,
+            n_skipped=confusion.n_skipped,
+        )
+        return NumericEvaluationResult(score=headline, details=details)