Add the metrics registry and the run report

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
     # trials each) are heavy and run in the nightly instead. No marker filter, so
     # both subprocess and unit tests run.
     env:
-      TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/"
+      TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/ autodata_tests/metrics/"
 
     steps:
       - name: nvidia-smi

autodata_metrics/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Offline metrics of AutoData datasets and generation runs.
+
+Producers record evidence; :func:`build_report` computes every registered metric from it without a simulator.
+Importing this package registers the built-in metrics.
+"""
+
+from autodata_metrics import viability
+from autodata_metrics.evidence import Episode, Evidence
+from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable, Value, metric
+from autodata_metrics.report import build_report, format_report, gate_violations

autodata_metrics/evidence.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""What a metric reads: a dataset's episodes and ``env_args``, and its generation run's record and attempt log."""
+
+from __future__ import annotations
+
+import h5py
+import json
+import numpy as np
+from collections.abc import Mapping
+from typing import Any
+
+
+def _has_path(record: Any, dotted_key: str) -> bool:
+    """Whether ``record[a][b]`` exists for ``dotted_key`` ``"a.b"``."""
+    for part in dotted_key.split("."):
+        if not isinstance(record, Mapping) or part not in record:
+            return False
+        record = record[part]
+    return True
+
+
+def read_env_args(h5_file: h5py.File) -> dict[str, Any]:
+    """The dataset's ``env_args``, or empty when they are not a JSON object."""
+    raw_env_args = h5_file["data"].attrs.get("env_args", "{}")
+    env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args)
+    return env_args if isinstance(env_args, dict) else {}
+
+
+class Episode:
+    """One episode of a dataset; arrays are read when accessed."""
+
+    def __init__(self, group: h5py.Group) -> None:
+        self.name = group.name.rsplit("/", 1)[-1]
+        self._group = group
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._group
+
+    def __getitem__(self, key: str) -> np.ndarray:
+        return self._group[key][()]
+
+
+class Evidence:
+    """A dataset, and optionally the generation run that produced it, as metrics read them.
+
+    Attributes:
+        episodes: The dataset's episodes.
+        env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer.
+        result: Generation result, or None for a dataset without a run record or an interrupted run.
+        run: The result's ``run`` block, or empty.
+        attempts: Attempt-log lines, or None.
+        source: Evidence of the source dataset the run started from, or None.
+    """
+
+    def __init__(
+        self,
+        h5_file: h5py.File,
+        result: Mapping[str, Any] | None = None,
+        attempts: list[dict[str, Any]] | None = None,
+        source: Evidence | None = None,
+    ) -> None:
+        self.env_args = read_env_args(h5_file)
+        self.episodes = [Episode(group) for group in h5_file["data"].values()]
+        self.result = result
+        self.run: Mapping[str, Any] = (result or {}).get("run", {})
+        self.attempts = attempts
+        self.source = source
+
+    def has(self, key: str) -> bool:
+        """Whether the evidence key ``key`` is present.
+
+        A key is one of:
+
+        - an HDF5 path present in every episode, such as ``step_success``;
+        - ``env_args.<path>`` or ``run.<path>``, a dotted path into ``env_args`` or the run block;
+        - ``attempts``, the attempt log, or ``attempts.<path>``, a dotted path present in every attempt line;
+        - ``source``, the source dataset, or ``source.<key>``, a key of the source dataset.
+        """
+        head, _, rest = key.partition(".")
+        if head == "source":
+            return self.source is not None and (not rest or self.source.has(rest))
+        if head == "attempts":
+            return bool(self.attempts) and (not rest or all(_has_path(line, rest) for line in self.attempts))
+        if head == "env_args":
+            return _has_path(self.env_args, rest)
+        if head == "run":
+            return _has_path(self.run, rest)
+        return bool(self.episodes) and all(key in episode for episode in self.episodes)

autodata_metrics/registry.py

@@ -0,0 +1,104 @@
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Registry of dataset and run metrics.
+
+A metric is a function of :class:`~autodata_metrics.evidence.Evidence` registered with :func:`metric`. It declares
+the evidence it ``requires``; the runner reports it as not applicable, naming the missing key, instead of calling it
+without that evidence.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from autodata_metrics.evidence import Evidence
+
+AREAS = ("validity", "diversity", "viability", "diagnostics")
+
+
+class NotApplicable(Exception):
+    """Raised by a metric whose evidence is present but does not let it apply."""
+
+
+@dataclass(frozen=True)
+class Value:
+    """What a metric computed.
+
+    Attributes:
+        value: A number, or named numbers such as percentiles.
+        n: Sample count the value rests on.
+        details: Context a reader needs to interpret the value.
+    """
+
+    value: float | dict[str, float]
+    n: int
+    details: Mapping[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MetricSpec:
+    """A registered metric and how to report it."""
+
+    name: str
+    compute: Callable[[Evidence], Value]
+    area: str
+    signal: str
+    scope: Literal["dataset", "run"]
+    unit: str
+    better: Literal["higher", "lower"] | None
+    requires: tuple[str, ...]
+    primary: str | None
+    version: int
+
+
+REGISTERED_METRICS: dict[str, MetricSpec] = {}
+
+
+def metric(
+    name: str,
+    *,
+    area: str,
+    signal: str,
+    scope: Literal["dataset", "run"],
+    unit: str,
+    better: Literal["higher", "lower"] | None = None,
+    requires: tuple[str, ...] | list[str] = (),
+    primary: str | None = None,
+    version: int = 1,
+) -> Callable[[Callable[[Evidence], Value]], Callable[[Evidence], Value]]:
+    """Register the decorated function as the metric ``name``.
+
+    Args:
+        name: Unique metric name, used in reports and gates.
+        area: One of :data:`AREAS`.
+        signal: What the metric measures within its area, e.g. ``"task correctness"``.
+        scope: ``"dataset"`` metrics read episodes; ``"run"`` metrics read attempts and the run record.
+        unit: Unit of the value.
+        better: Which direction is better, if either.
+        requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`.
+        primary: For a dict value, the entry a gate checks by default.
+        version: Raise it whenever the definition changes, so gates pinned to the old one fail.
+    """
+    assert area in AREAS, f"Metric {name!r} has area {area!r}; expected one of {AREAS}."
+
+    def register(compute: Callable[[Evidence], Value]) -> Callable[[Evidence], Value]:
+        assert name not in REGISTERED_METRICS, f"Metric {name!r} is already registered."
+        REGISTERED_METRICS[name] = MetricSpec(
+            name=name,
+            compute=compute,
+            area=area,
+            signal=signal,
+            scope=scope,
+            unit=unit,
+            better=better,
+            requires=tuple(requires),
+            primary=primary,
+            version=version,
+        )
+        return compute
+
+    return register