From a058fd8e45f6ef4c804a5f75b2f22a023cd4126b Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 13:54:04 -0500 Subject: [PATCH 01/24] feat: record metric inputs and run metadata during generation Generation, annotation and retargeting now record what dataset metrics need, without computing any metric: - Every producer records step_success from the task's success term. Generation also records the per-step EEF and commanded target poses, but not object poses, which annotation keeps for the source pool. - Each exported dataset gets an autodata entry in env_args: run id, exported count, pose-action layout, robot joint names with simulator limits, and optional datasheet hardware_limits from the embodiment. - --result_file adds a run block (run id, seed, revision, input digests, hardware, setup and generation time) and an attempt log, one line per attempt with its failure cause and algorithm extras. - GenerationAlgorithm.attempt_extras() lets an algorithm report per-attempt facts; SkillGen reports its motion-planning time. --- autodata_core/algorithms.py | 17 ++- autodata_core/data_generator.py | 25 ++++- .../embodiments/franka_ik_rel.yaml | 13 +++ .../embodiments/franka_ik_rel_skillgen.yaml | 13 +++ .../absolute_pose_embodiment_adapter.py | 9 ++ .../bimanual_embodiment_adapter.py | 8 ++ .../embodiments/embodiment_adapter.py | 9 ++ autodata_interfaces/embodiments/factory.py | 3 +- .../single_arm_embodiment_adapter.py | 6 ++ autodata_interfaces/env/__init__.py | 2 + .../env/datagen_info_recorder.py | 79 ++++++++++++++ .../env/isaaclab_env_interface.py | 37 ++++++- autodata_retargeting/runner.py | 9 +- autodata_tests/core/test_data_generator.py | 17 ++- autodata_tests/e2e/test_annotation.py | 4 +- .../e2e/test_mimicgen_data_generation.py | 6 +- .../e2e/test_skillgen_bin_data_generation.py | 5 +- autodata_tests/utils/utils.py | 41 +++++++ autodata_utils/dataset_metadata.py | 38 +++++++ autodata_utils/generation_result.py | 51 +++++++++ scripts/annotate_demos.py | 61 +++++------ scripts/generate_dataset.py | 101 +++++++++++++++++- .../references/cli-reference.md | 2 +- 23 files changed, 507 insertions(+), 49 deletions(-) create mode 100644 autodata_interfaces/env/datagen_info_recorder.py create mode 100644 autodata_utils/dataset_metadata.py diff --git a/autodata_core/algorithms.py b/autodata_core/algorithms.py index 65d6daf..8c969be 100644 --- a/autodata_core/algorithms.py +++ b/autodata_core/algorithms.py @@ -15,7 +15,8 @@ from __future__ import annotations -from collections.abc import Iterator +import time +from collections.abc import Iterable, Iterator from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -71,6 +72,14 @@ def validate_setup(self, datastream: Datastream) -> None: or ``datastream.get_env()`` (escape hatch) to enforce algorithm-specific invariants. """ + def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: + """Algorithm-specific facts about one finished attempt, written to its attempt-log line. Default: none. + + Args: + eef_states: Per-EEF generation state of the attempt. + """ + return {} + def plan_subtask_trajectory( self, *, @@ -190,6 +199,10 @@ def __init__(self, motion_planners: dict[int, Any]) -> None: assert motion_planners, "SkillGen requires at least one motion planner" self.motion_planners = motion_planners + def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: + """Report the attempt's wall-clock motion-planning time [s] as ``planning_time_s``.""" + return {"planning_time_s": sum(state.planning_time_s for state in eef_states)} + def plan_subtask_trajectory( self, *, @@ -238,6 +251,7 @@ def plan_subtask_trajectory( ) planner = self.motion_planners[env_id] + start_time = time.perf_counter() planning_success = planner.update_world_and_plan_motion( target_pose=target_pose, expected_attached_object=expected_attached_object, @@ -245,6 +259,7 @@ def plan_subtask_trajectory( step_size=getattr(planner, "step_size", None), enable_retiming=(hasattr(planner, "step_size") and planner.step_size is not None), ) + eef_state.planning_time_s += time.perf_counter() - start_time if not planning_success: return None diff --git a/autodata_core/data_generator.py b/autodata_core/data_generator.py index 59481cb..59d1e84 100644 --- a/autodata_core/data_generator.py +++ b/autodata_core/data_generator.py @@ -15,7 +15,7 @@ import torch from copy import deepcopy from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal from isaaclab.managers import TerminationTermCfg @@ -47,10 +47,21 @@ async def _optional_lock(lock: asyncio.Lock | None): @dataclass(frozen=True) class GenerationResult: - """Output of one :meth:`DataGenerator.generate` invocation.""" + """Output of one :meth:`DataGenerator.generate` invocation. + + Attributes: + initial_state: Scene state of every env right after this env's reset. + success: Whether the task succeeded at any step of the attempt. + failure_cause: ``"planner"`` when motion planning aborted the attempt, otherwise ``"execution"`` + for a failed attempt and None on success. + extras: Algorithm-specific facts about the attempt, from + :meth:`~autodata_core.algorithms.GenerationAlgorithm.attempt_extras`. + """ initial_state: dict success: bool + failure_cause: Literal["planner", "execution"] | None = None + extras: dict[str, Any] = field(default_factory=dict) @dataclass @@ -76,6 +87,7 @@ class _EEFGenerationState: # subtask trajectory here so the follow-up call splices it in. pending_subtask_trajectory: WaypointTrajectory | None = None is_in_motion_plan: bool = False + planning_time_s: float = 0.0 class DataGenerator: @@ -569,7 +581,12 @@ async def generate( ) if result is None: # Planning failure (e.g. SkillGen motion planner). Abort with no success. - return GenerationResult(initial_state=initial_state, success=False) + return GenerationResult( + initial_state=initial_state, + success=False, + failure_cause="planner", + extras=self.algorithm.attempt_extras(eef_states.values()), + ) next_trajectory, _is_motion_plan = result eef_state.current_trajectory = next_trajectory eef_state.subtask_step_index = 0 @@ -620,6 +637,8 @@ async def generate( return GenerationResult( initial_state=initial_state, success=buffers.success, + failure_cause=None if buffers.success else "execution", + extras=self.algorithm.attempt_extras(eef_states.values()), ) # ------------------------------------------------------------------ diff --git a/autodata_examples/embodiments/franka_ik_rel.yaml b/autodata_examples/embodiments/franka_ik_rel.yaml index 2ad1e97..3194ae2 100644 --- a/autodata_examples/embodiments/franka_ik_rel.yaml +++ b/autodata_examples/embodiments/franka_ik_rel.yaml @@ -31,3 +31,16 @@ action_layout: # that same frame, so no shift is needed. SkillGen's cube-stack dataset is annotated in the # panda_hand frame, so it needs [0, 0, 0.1034] — use franka_cube_stack_embodiment_skillgen.yaml. eef_offset: [0.0, 0.0, 0.0] + +# Franka FCI datasheet joint limits, keyed by joint-name pattern. Read only by the dataset metrics. +hardware_limits: + velocity: # [rad/s] + panda_joint[1-4]: 2.175 + panda_joint[5-7]: 2.61 + acceleration: # [rad/s^2] + panda_joint1: 15.0 + panda_joint2: 7.5 + panda_joint3: 10.0 + panda_joint4: 12.5 + panda_joint5: 15.0 + panda_joint[6-7]: 20.0 diff --git a/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml b/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml index 5e27381..618a461 100644 --- a/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml +++ b/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml @@ -35,3 +35,16 @@ action_layout: # approach axis) while the SkillGen dataset is annotated in the panda_hand frame, so the adapter # shifts reports back to panda_hand to keep planned transits and recorded skill segments aligned. eef_offset: [0.0, 0.0, 0.1034] + +# Franka FCI datasheet joint limits, keyed by joint-name pattern. Read only by the dataset metrics. +hardware_limits: + velocity: # [rad/s] + panda_joint[1-4]: 2.175 + panda_joint[5-7]: 2.61 + acceleration: # [rad/s^2] + panda_joint1: 15.0 + panda_joint2: 7.5 + panda_joint3: 10.0 + panda_joint4: 12.5 + panda_joint5: 15.0 + panda_joint[6-7]: 20.0 diff --git a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py index c1d6c09..33cc04c 100644 --- a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py @@ -77,6 +77,15 @@ def from_dict(cls, data: dict[str, Any]) -> AbsolutePoseEmbodimentAdapter: """Build the adapter from a validated descriptor.""" return cls(**data) + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns of every EEF that commands a pose; they are not clipped.""" + pose_action = { + name: {"columns": list(spec["pose_action_slice"]), "bound": None} + for name, spec in self.eefs.items() + if spec.get("pose_action_slice") is not None + } + return {**super().metric_context(), "pose_action": pose_action} + def get_eef_names(self) -> tuple[str, ...]: return tuple(self.eefs) diff --git a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py index 206cc86..ad698e4 100644 --- a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py @@ -191,6 +191,14 @@ def action_dim(self) -> int: channel_ends = [channel_slice[1] for channel_slice in self.passthrough_channels.values()] return max([self.hand_joints_slice[1], *channel_ends]) + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns of both hands; they are not clipped.""" + pose_action = { + self.left.name: {"columns": list(self.left_pose_slice), "bound": None}, + self.right.name: {"columns": list(self.right_pose_slice), "bound": None}, + } + return {**super().metric_context(), "pose_action": pose_action} + def action_to_target_eef_pose(self, action: torch.Tensor) -> dict[str, torch.Tensor]: """Convert env action to target EEF poses for both arms. diff --git a/autodata_interfaces/embodiments/embodiment_adapter.py b/autodata_interfaces/embodiments/embodiment_adapter.py index f230bfe..a219c6b 100644 --- a/autodata_interfaces/embodiments/embodiment_adapter.py +++ b/autodata_interfaces/embodiments/embodiment_adapter.py @@ -50,6 +50,15 @@ class EmbodimentAdapter(ABC): robot_asset_name: str = "robot" raw_config: dict[str, Any] | None = None + def metric_context(self) -> dict[str, Any]: + """Facts about this embodiment that the dataset metrics read, stored with every dataset it records. + + Subclasses add their pose-action layout. The base adds the optional ``hardware_limits`` section of the + embodiment config (datasheet joint limits, keyed by joint-name pattern). + """ + hardware_limits = (self.raw_config or {}).get("hardware_limits") + return {"hardware_limits": hardware_limits} if hardware_limits else {} + def get_navigation_eef_name(self) -> str | None: """Return the configured navigation channel without requiring a live environment.""" diff --git a/autodata_interfaces/embodiments/factory.py b/autodata_interfaces/embodiments/factory.py index fb1cc98..6267e59 100644 --- a/autodata_interfaces/embodiments/factory.py +++ b/autodata_interfaces/embodiments/factory.py @@ -70,7 +70,8 @@ def embodiment_adapter_from_dict(data: dict[str, Any], env: Any = None) -> Embod type_key in EMBODIMENT_TYPE_REGISTRY ), f"Unknown embodiment type {type_key!r}. Registered: {sorted(EMBODIMENT_TYPE_REGISTRY)}" cls = EMBODIMENT_TYPE_REGISTRY[type_key] - payload = {k: v for k, v in data.items() if k != "type"} + # ``hardware_limits`` is read from ``raw_config`` by :meth:`EmbodimentAdapter.metric_context`. + payload = {k: v for k, v in data.items() if k not in ("type", "hardware_limits")} adapter = cls.from_dict(payload) # type: ignore[attr-defined] adapter.raw_config = data # the full parsed config, for sections no typed adapter field models if env is not None: diff --git a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py index 2c332a0..3b25dec 100644 --- a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py @@ -133,6 +133,12 @@ class DeltaPoseIKSingleArmAdapter(SingleArmEmbodimentAdapter): clip_pose_action_to_unit: bool = True + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns and their clip bound.""" + bound = 1.0 if self.clip_pose_action_to_unit else None + pose_action = {self.eef_name: {"columns": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} + return {**super().metric_context(), "pose_action": pose_action} + @property def action_dim(self) -> int: """Total width of the env action vector.""" diff --git a/autodata_interfaces/env/__init__.py b/autodata_interfaces/env/__init__.py index a673b88..b75c594 100644 --- a/autodata_interfaces/env/__init__.py +++ b/autodata_interfaces/env/__init__.py @@ -6,6 +6,7 @@ from autodata_interfaces.env.env_profile import EnvironmentProfile from autodata_interfaces.env.isaaclab_env_interface import ( apply_env_profile, + collect_autodata_entry, env_loop, get_env_name_from_dataset, setup_env_config, @@ -18,5 +19,6 @@ "get_env_name_from_dataset", "setup_output_paths", "env_loop", + "collect_autodata_entry", "setup_env_config", ] diff --git a/autodata_interfaces/env/datagen_info_recorder.py b/autodata_interfaces/env/datagen_info_recorder.py new file mode 100644 index 0000000..15bb80b --- /dev/null +++ b/autodata_interfaces/env/datagen_info_recorder.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Recorder terms that store the per-step data annotation, generation and the dataset metrics read.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg +from isaaclab.managers import RecorderTerm, RecorderTermCfg, TerminationTermCfg +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from autodata_interfaces.datastream import Datastream + + +class PreStepDatagenInfoRecorder(RecorderTerm): + """Records end-effector and commanded target poses, and optionally object poses, before each step. + + Attributes: + datastream: Datastream to read. Set it on the class once the Datastream exists, since Isaac Lab + copies recorder configurations when building the env. + """ + + datastream: Datastream | None = None + + def record_pre_step(self): + assert self.datastream is not None, "Set PreStepDatagenInfoRecorder.datastream before stepping the env." + datagen_info = { + "eef_pose": self.datastream.embodiment_adapter.get_eef_poses(env_ids=None), + "target_eef_pose": self.datastream.action_to_target_eef_pose(self._env.action_manager.action), + } + if self.cfg.record_object_poses: + datagen_info["object_pose"] = self.datastream.get_object_poses() + return "obs/datagen_info", datagen_info + + +@configclass +class PreStepDatagenInfoRecorderCfg(RecorderTermCfg): + """Configuration for :class:`PreStepDatagenInfoRecorder`.""" + + class_type: type[RecorderTerm] = PreStepDatagenInfoRecorder + record_object_poses: bool = True + """Annotation needs object poses for the source pool; generation leaves them out, as no metric reads them.""" + + +class PostStepSuccessRecorder(RecorderTerm): + """Records whether the task's success term holds after each step, as ``step_success``.""" + + def record_post_step(self): + success_term = self.cfg.success_term + return "step_success", success_term.func(self._env, **success_term.params) + + +@configclass +class PostStepSuccessRecorderCfg(RecorderTermCfg): + """Configuration for :class:`PostStepSuccessRecorder`.""" + + class_type: type[RecorderTerm] = PostStepSuccessRecorder + success_term: TerminationTermCfg | None = None + """The task's success term; the env itself runs without termination terms.""" + + +@configclass +class DatagenInfoRecorderManagerCfg(ActionStateRecorderManagerCfg): + """Action/state recorder plus the per-step ``obs/datagen_info`` term.""" + + record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() + + +def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg) -> None: + """Add the ``step_success`` term to any recorder manager configuration. + + Args: + recorder_cfg: Recorder manager configuration the env will be built with. + success_term: The task's success term. + """ + recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) diff --git a/autodata_interfaces/env/isaaclab_env_interface.py b/autodata_interfaces/env/isaaclab_env_interface.py index 3a8f916..80754f0 100644 --- a/autodata_interfaces/env/isaaclab_env_interface.py +++ b/autodata_interfaces/env/isaaclab_env_interface.py @@ -18,7 +18,7 @@ import contextlib import os import torch -from typing import Any +from typing import TYPE_CHECKING, Any import isaaclab.sim as sim_utils from isaaclab.assets import RigidObjectCfg @@ -32,9 +32,43 @@ from isaaclab_physx.sim.schemas import PhysxRigidBodyPropertiesCfg from isaaclab_tasks.utils.parse_cfg import parse_env_cfg +from autodata_interfaces.env.datagen_info_recorder import add_step_success_recorder from autodata_interfaces.env.env_profile import EnvironmentProfile, convert_event_params from autodata_interfaces.env.pink_ik_action import PinkInverseKinematicsActionSharedUrdf from autodata_interfaces.tasks.generation_policy_spec import GenerationPolicy +from autodata_utils.dataset_metadata import AUTODATA_ENTRY_VERSION +from autodata_utils.tensor_utils import as_torch + +if TYPE_CHECKING: + from autodata_interfaces.embodiments.embodiment_adapter import EmbodimentAdapter + + +def collect_autodata_entry(env: Any, embodiment_adapter: EmbodimentAdapter, run_id: str) -> dict[str, Any]: + """The ``autodata`` entry of the datasets this env exports, read while the env is alive. + + It holds the run id, the embodiment's metric context, and the robot's joint names with env 0's simulator + position limits [rad or m] and velocity limits [rad/s or m/s]. Unbounded limits are stored as None. + :func:`~autodata_utils.dataset_metadata.merge_autodata_entry` writes it once the env has closed the file. + + Args: + env: Live environment. + embodiment_adapter: Adapter of the embodiment the env drives. + run_id: Identifier shared by everything this run writes. + """ + entry: dict[str, Any] = {"version": AUTODATA_ENTRY_VERSION, "run_id": run_id, **embodiment_adapter.metric_context()} + robot = env.scene.articulations.get(embodiment_adapter.robot_asset_name) + if robot is not None: + entry["joints"] = { + "names": list(robot.joint_names), + "position_limits": _finite_or_none(as_torch(robot.data.joint_pos_limits)[0].cpu()), + "velocity_limits": _finite_or_none(as_torch(robot.data.joint_vel_limits)[0].cpu()), + } + return entry + + +def _finite_or_none(values: torch.Tensor) -> list: + """Nested lists of ``values`` with non-finite entries as None, so the JSON stays standard.""" + return [_finite_or_none(row) if row.dim() else (row.item() if torch.isfinite(row) else None) for row in values] def setup_output_paths(output_file_path: str) -> tuple[str, str]: @@ -231,6 +265,7 @@ def setup_env_config( env_cfg.recorders = ActionStateRecorderManagerCfg() else: env_cfg.recorders = recorder_cfg + add_step_success_recorder(env_cfg.recorders, success_term) env_cfg.recorders.dataset_export_dir_path = output_dir env_cfg.recorders.dataset_filename = output_file_name diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index f1e9ddb..e4a05d2 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -7,13 +7,15 @@ import numpy as np import random import torch +import uuid from copy import deepcopy from isaaclab.utils.datasets import HDF5DatasetFileHandler from autodata_interfaces.embodiments import embodiment_adapter_from_yaml -from autodata_interfaces.env import setup_env_config, setup_output_paths +from autodata_interfaces.env import collect_autodata_entry, setup_env_config, setup_output_paths from autodata_interfaces.tasks.generation_policy_spec import GenerationPolicy +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry from .config import RetargetConfig from .eef_offset import build_eef_offsets, compose_retarget_eef_offsets_from_adapters @@ -117,8 +119,10 @@ def run(args, simulation_app) -> int: torch.manual_seed(generation_policy.seed) dataset_file_handler = None + autodata_entry = None try: target_adapter.bind_env(env) + autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) env.reset() # Snapshot the target's home scene state once; each replay restores its robot from here # (deepcopy so later sim steps cannot mutate the cached tensors). @@ -180,3 +184,6 @@ def run(args, simulation_app) -> int: if dataset_file_handler is not None: dataset_file_handler.close() env.close() + if autodata_entry is not None: + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) diff --git a/autodata_tests/core/test_data_generator.py b/autodata_tests/core/test_data_generator.py index 806c0e9..6667ca1 100644 --- a/autodata_tests/core/test_data_generator.py +++ b/autodata_tests/core/test_data_generator.py @@ -196,6 +196,21 @@ def test_navigation_does_not_skip_playback_after_cancellation_or_while_paused(pa assert all(state.subtask_step_index == 4 for state in states.values()) +def test_planning_failure_is_attributed_to_the_planner(monkeypatch): + generator = _navigation_generator(active=False, reached=True) + generator.datastream.env = SimpleNamespace(recorder_manager=Mock()) + monkeypatch.setattr(generator.algorithm, "plan_subtask_trajectory", lambda **kwargs: None) + monkeypatch.setattr(generator, "_reset_environment_for_generation", AsyncMock(return_value=(torch.tensor([0]), {}))) + + result = asyncio.run( + generator.generate( + env_id=0, success_term=None, env_reset_queue=asyncio.Queue(), env_action_queue=asyncio.Queue() + ) + ) + + assert result.success is False and result.failure_cause == "planner" + + def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): generator = _navigation_generator(active=True, reached=False) generator.datastream.generation_policy.max_navigation_hold_steps = 3 @@ -219,7 +234,7 @@ def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): ) ) - assert result.success is False + assert result.success is False and result.failure_cause == "execution" assert execute.await_count == 3 assert recorder.set_success_to_episodes.call_args.args[1].item() is False recorder.export_episodes.assert_called_once() diff --git a/autodata_tests/e2e/test_annotation.py b/autodata_tests/e2e/test_annotation.py index f5383d3..bedc99c 100644 --- a/autodata_tests/e2e/test_annotation.py +++ b/autodata_tests/e2e/test_annotation.py @@ -12,7 +12,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_episodes, assert_valid_dataset @pytest.mark.with_subprocess @@ -38,6 +38,8 @@ def test_automatic_annotation_from_dataset(tmp_path): str(output_file), ]) assert_valid_dataset(str(output_file), min_num_demos=1) + entry = assert_recorded_episodes(str(output_file)) + assert entry["annotation"]["annotated_episodes"] == entry["exported_episodes"] with h5py.File(input_file, "r") as source, h5py.File(output_file, "r") as annotated: assert len(annotated["data"]) == len(source["data"]) assert json.loads(annotated["data"].attrs["env_args"])["env_name"] == "IsaacContrib-Stack-Cube-Franka-IK-Rel" diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index cfb02b9..a97271e 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -8,7 +8,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -19,6 +19,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: with tempfile.TemporaryDirectory() as temp_dir: output_file = os.path.join(temp_dir, "generated.hdf5") + result_file = os.path.join(temp_dir, "generation_result.json") args = [ TestPaths.python_path, @@ -35,6 +36,8 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: os.path.join(TestPaths.test_data_dir, "annotated_dataset_franka_stack_mimicgen.hdf5"), "--output_file", output_file, + "--result_file", + result_file, "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", @@ -47,6 +50,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + assert_recorded_run(output_file, result_file) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 0627f18..90b5a17 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -17,7 +17,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -62,6 +62,9 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + attempts = assert_recorded_run(output_file, result_file) + # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. + assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index f9cbde4..dd528a3 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -41,3 +41,44 @@ def read_generation_result(result_file: str) -> tuple[int, int]: assert num_attempts > 0, f"Generation result has invalid num_attempts: {num_attempts}." assert num_success + num_failures == num_attempts, "Generation result counts do not add up." return num_success, num_attempts + + +def assert_recorded_episodes(dataset_file: str) -> dict: + """Assert the per-step recording and the ``autodata`` entry agree with the dataset's episodes. + + Every episode has one ``step_success`` value per action, its success attribute equals whether + ``step_success`` ever held, and the entry counts every exported episode. + + Returns: + The dataset's ``autodata`` entry. + """ + + with h5py.File(dataset_file, "r") as h5_file: + data = h5_file["data"] + entry = json.loads(data.attrs["env_args"])["autodata"] + assert entry["exported_episodes"] == len(data), f"The entry counts {entry['exported_episodes']} episodes." + for name, episode in data.items(): + step_success = episode["step_success"][:] + assert len(step_success) == len(episode["actions"]), f"{name}: step_success and actions differ in length." + assert bool(step_success.any()) == bool(episode.attrs["success"]), f"{name}: step_success disagrees." + return entry + + +def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: + """Assert a generation run's dataset, result JSON and attempt log describe the same run. + + Returns: + The attempt-log lines. + """ + + entry = assert_recorded_episodes(dataset_file) + with open(result_file, encoding="utf-8") as result_handle: + result = json.load(result_handle) + attempts_file = os.path.join(os.path.dirname(result_file), result["run"]["attempts_file"]) + with open(attempts_file, encoding="utf-8") as attempts_handle: + attempts = [json.loads(line) for line in attempts_handle if line.strip()] + assert entry["exported_episodes"] == result["num_success"], "The dataset and result count different successes." + assert len(attempts) == result["num_attempts"], "The attempt log and result count different attempts." + run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} + assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" + return attempts diff --git a/autodata_utils/dataset_metadata.py b/autodata_utils/dataset_metadata.py new file mode 100644 index 0000000..745077b --- /dev/null +++ b/autodata_utils/dataset_metadata.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The ``autodata`` entry of a dataset's ``env_args``: what the dataset metrics need to know about its making.""" + +from __future__ import annotations + +import h5py +import json +import os +from collections.abc import Mapping +from typing import Any + +AUTODATA_ENTRY_VERSION = 1 + + +def exported_dataset_files(output_dir: str, output_file_name: str) -> list[str]: + """The HDF5 files a recorder exported: the dataset and, when failed episodes were kept, its ``_failed`` file.""" + candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in ("", "_failed")] + return [path for path in candidates if os.path.exists(path)] + + +def merge_autodata_entry(dataset_path: str, entry: Mapping[str, Any]) -> None: + """Store ``entry`` under ``autodata`` in the dataset's ``env_args``, with the number of exported episodes. + + Call it after the env has closed the file. The episode count lets a reader detect a dataset merged from + several runs, since merging keeps the first file's ``env_args``. + + Args: + dataset_path: Exported HDF5 dataset. + entry: The entry, as built by :func:`~autodata_interfaces.env.collect_autodata_entry`. + """ + with h5py.File(dataset_path, "r+") as h5_file: + data = h5_file["data"] + raw_env_args = data.attrs.get("env_args", "{}") + env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + env_args["autodata"] = {**entry, "exported_episodes": len(data)} + data.attrs["env_args"] = json.dumps(env_args) diff --git a/autodata_utils/generation_result.py b/autodata_utils/generation_result.py index c38213d..090d4db 100644 --- a/autodata_utils/generation_result.py +++ b/autodata_utils/generation_result.py @@ -6,11 +6,15 @@ from __future__ import annotations import contextlib +import hashlib import json import os +import shutil +import subprocess import tempfile from collections.abc import Mapping from pathlib import Path +from typing import Any def write_generation_result( @@ -19,6 +23,7 @@ def write_generation_result( requested_trials: int, stats: Mapping[str, int], env_profile: Mapping[str, str | None] | None = None, + run_metadata: Mapping[str, Any] | None = None, ) -> None: """Atomically write the final outcome of a completed data-generation run. @@ -31,6 +36,7 @@ def write_generation_result( during generation, or None when the run used the unmodified base task. Recorded here because the generated dataset stores only the base env id and would otherwise not reveal that the scene was modified. + run_metadata: How the run was configured and how long it took, stored under ``run``. """ required_stats = ("num_success", "num_failures", "num_attempts") for stat_name in required_stats: @@ -51,6 +57,8 @@ def write_generation_result( } if env_profile is not None: result["env_profile"] = dict(env_profile) + if run_metadata is not None: + result["run"] = dict(run_metadata) file_descriptor, temporary_file_name = tempfile.mkstemp( prefix=f".{result_path.name}.", suffix=".tmp", dir=result_path.parent, text=True @@ -65,3 +73,46 @@ def write_generation_result( finally: with contextlib.suppress(FileNotFoundError): os.unlink(temporary_file_name) + + +def attempts_file_for(result_file: str) -> str: + """Path of the attempt log written next to ``result_file``.""" + return str(Path(result_file).with_suffix(".attempts.jsonl")) + + +class AttemptLog: + """Appends one JSON line per completed generation attempt, stamped with the run id.""" + + def __init__(self, path: str, run_id: str) -> None: + self._path = Path(path) + self._run_id = run_id + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text("", encoding="utf-8") + + def write(self, record: Mapping[str, Any]) -> None: + with self._path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"run_id": self._run_id, **record}, sort_keys=True) + "\n") + + +def input_record(path: str | None) -> dict[str, str] | None: + """Path and SHA-256 of a run input, or None when the input was not given.""" + if path is None: + return None + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return {"path": path, "sha256": digest.hexdigest()} + + +def code_revision() -> str | None: + """Git commit of the AutoData checkout, or None without git or outside a checkout.""" + if shutil.which("git") is None: + return None + result = subprocess.run( + ["git", "-C", str(Path(__file__).resolve().parent), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else None diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index b0d6b2b..d27984c 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -104,23 +104,33 @@ import math # noqa: E402 import os # noqa: E402 import sys # noqa: E402 +import time # noqa: E402 import torch # noqa: E402 import traceback # noqa: E402 +import uuid # noqa: E402 from collections.abc import Callable # noqa: E402 import isaaclab_tasks # noqa: F401, E402 (registers gym envs) from isaaclab.devices import Se3Keyboard, Se3KeyboardCfg # noqa: E402 -from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg # noqa: E402 -from isaaclab.managers import DatasetExportMode, RecorderTerm, RecorderTermCfg, TerminationTermCfg # noqa: E402 -from isaaclab.utils import configclass # noqa: E402 +from isaaclab.managers import DatasetExportMode, TerminationTermCfg # noqa: E402 from isaaclab.utils.datasets import EpisodeData, HDF5DatasetFileHandler # noqa: E402 from autodata_core.pool import DataGenInfoPool # noqa: E402 from autodata_interfaces.datastream import Datastream # noqa: E402 from autodata_interfaces.embodiments import embodiment_adapter_from_yaml # noqa: E402 -from autodata_interfaces.env import get_env_name_from_dataset, setup_env_config, setup_output_paths # noqa: E402 +from autodata_interfaces.env import ( # noqa: E402 + collect_autodata_entry, + get_env_name_from_dataset, + setup_env_config, + setup_output_paths, +) +from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 + DatagenInfoRecorderManagerCfg, + PreStepDatagenInfoRecorder, +) from autodata_interfaces.env.registration import register_external_environment # noqa: E402 from autodata_interfaces.tasks.task_descriptor import TaskDescriptor # noqa: E402 +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 is_paused = False current_action_index = 0 @@ -151,34 +161,6 @@ def mark_subtask_cb() -> None: print(f"Marked a subtask signal at action index: {current_action_index}") -# Recorder term that writes per-step datagen info read through the Datastream. -class PreStepDatagenInfoRecorder(RecorderTerm): - """Recorder term that records ``obs/datagen_info`` from the Datastream on each step.""" - - def record_pre_step(self): - assert _datastream is not None, "Datastream must be initialized before recording." - datagen_info = { - "object_pose": _datastream.get_object_poses(), - "eef_pose": _datastream.embodiment_adapter.get_eef_poses(env_ids=None), - "target_eef_pose": _datastream.action_to_target_eef_pose(self._env.action_manager.action), - } - return "obs/datagen_info", datagen_info - - -@configclass -class PreStepDatagenInfoRecorderCfg(RecorderTermCfg): - """Configuration for :class:`PreStepDatagenInfoRecorder`.""" - - class_type: type[RecorderTerm] = PreStepDatagenInfoRecorder - - -@configclass -class AnnotationRecorderManagerCfg(ActionStateRecorderManagerCfg): - """Action/state recorder plus the datagen-info term used during annotation.""" - - record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() - - def _build_signal_names( task_descriptor: TaskDescriptor, annotate_start_signals: bool, @@ -264,13 +246,14 @@ def main() -> int: num_envs=1, device=args_cli.device, generation_policy_params=generation_policy, - recorder_cfg=AnnotationRecorderManagerCfg(), + recorder_cfg=DatagenInfoRecorderManagerCfg(), enable_cameras=args_cli.record_images, ) # Only export episodes we explicitly mark successful (i.e. fully annotated). env_cfg.recorders.dataset_export_mode = DatasetExportMode.EXPORT_SUCCEEDED_ONLY env = gym.make(env_name, cfg=env_cfg).unwrapped + autodata_entry = None try: # Create the Datastream embodiment_adapter = embodiment_adapter_from_yaml(args_cli.embodiment) @@ -287,6 +270,8 @@ def main() -> int: source_pool=empty_pool, uses_start_signals=annotate_start_signals, ) + PreStepDatagenInfoRecorder.datastream = _datastream + autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) env.reset() @@ -311,6 +296,7 @@ def main() -> int: exported_episode_count = 0 processed_episode_count = 0 + annotation_start = time.monotonic() with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(): for episode_index, episode_name in enumerate(episode_names): if not simulation_app.is_running() or simulation_app.is_exiting(): @@ -342,10 +328,19 @@ def main() -> int: f"\nExported {exported_episode_count} (out of {processed_episode_count}) annotated" f" episode{'s' if exported_episode_count != 1 else ''} to {args_cli.output_file}." ) + autodata_entry["annotation"] = { + "input_episodes": len(episode_names), + "processed_episodes": processed_episode_count, + "annotated_episodes": exported_episode_count, + "time_s": time.monotonic() - annotation_start, + } print("Exiting the app.") return exported_episode_count finally: env.close() + if autodata_entry is not None: + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) def replay_episode( diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index d6f5167..5a3bd48 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -33,6 +33,7 @@ """Launch Isaac Sim Simulator first.""" import argparse +import time from isaaclab.app import AppLauncher @@ -42,6 +43,8 @@ from autodata_interfaces.tasks.task_descriptor_utils import resolve_task_algorithm from autodata_utils.cli import callable_path +PROCESS_START = time.monotonic() + # Hardcoded to keep argparse importable without pulling in the heavy core package. # Add new algorithms here when registering them in autodata_core.algorithms. _ALG_CHOICES = ["mimicgen", "dexmimicgen", "skillgen"] @@ -97,7 +100,10 @@ "--result_file", type=str, default=None, - help="Optional JSON path for the final structured generation result.", + help=( + "Optional JSON path for the final structured generation result. An attempt log with one JSON " + "line per attempt is written next to it (.attempts.jsonl)." + ), ) parser.add_argument( "--pause_subtask", @@ -137,24 +143,41 @@ import asyncio # noqa: E402 import gymnasium as gym # noqa: E402 import numpy as np # noqa: E402 +import platform # noqa: E402 import random # noqa: E402 import sys # noqa: E402 import torch # noqa: E402 import traceback # noqa: E402 +import uuid # noqa: E402 +from pathlib import Path # noqa: E402 from typing import Any # noqa: E402 from autodata_core import DataGenerator, get_algorithm # noqa: E402 from autodata_core.algorithms import REGISTERED_ALGORITHMS # noqa: E402 +from autodata_core.data_generator import GenerationResult # noqa: E402 from autodata_interfaces.datastream import Datastream # noqa: E402 from autodata_interfaces.env import ( # noqa: E402 EnvironmentProfile, + collect_autodata_entry, env_loop, get_env_name_from_dataset, setup_env_config, setup_output_paths, ) +from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 + DatagenInfoRecorderManagerCfg, + PreStepDatagenInfoRecorder, +) from autodata_interfaces.env.registration import load_callable, register_external_environment # noqa: E402 -from autodata_utils.generation_result import write_generation_result # noqa: E402 +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 +from autodata_utils.generation_result import ( # noqa: E402 + AttemptLog, + attempts_file_for, + code_revision, + input_record, + write_generation_result, +) +from autodata_utils.tensor_utils import as_torch # noqa: E402 async def run_data_generator( @@ -166,9 +189,11 @@ async def run_data_generator( success_term, pause_subtask: bool, stats: dict, + attempt_log: AttemptLog | None, ) -> None: - """Repeatedly call ``data_generator.generate`` and tally outcomes into ``stats``.""" + """Repeatedly call ``data_generator.generate``, tally outcomes into ``stats`` and log each attempt.""" while True: + attempt_start = time.monotonic() try: result = await data_generator.generate( env_id=env_id, @@ -187,6 +212,45 @@ async def run_data_generator( else: stats["num_failures"] += 1 stats["num_attempts"] += 1 + if attempt_log is not None: + attempt_log.write(_attempt_record(env_id, result, attempt_start, time.monotonic())) + + +def _attempt_record(env_id: int, result: GenerationResult, start_time: float, end_time: float) -> dict: + """One attempt-log line; times [s] are monotonic-clock readings.""" + return { + "env_id": env_id, + "success": result.success, + "failure_cause": result.failure_cause, + "start_time_s": start_time, + "end_time_s": end_time, + "initial_object_positions": { + name: as_torch(state["root_pose"])[env_id, :2].tolist() + for name, state in result.initial_state.get("rigid_object", {}).items() + }, + "extras": result.extras, + } + + +def _hardware(device: str) -> dict[str, str | None]: + """Model names of the GPU and CPU the run used; physics runs on the CPU when ``device`` is ``cpu``.""" + gpu = None + if torch.cuda.is_available(): + index = torch.device(device).index if device.startswith("cuda") else None + gpu = torch.cuda.get_device_name(torch.cuda.current_device() if index is None else index) + cpu = None + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.exists(): + model_lines = (line for line in cpuinfo.read_text().splitlines() if line.startswith("model name")) + cpu = next((line.split(":", 1)[1].strip() for line in model_lines), None) + return {"gpu": gpu, "num_gpus": 1 if gpu else 0, "cpu": cpu or platform.processor() or None} + + +def _generation_recorder_cfg() -> DatagenInfoRecorderManagerCfg: + """Actions and states, plus the per-step poses the dataset metrics read; object poses are left out.""" + recorder_cfg = DatagenInfoRecorderManagerCfg() + recorder_cfg.record_pre_step_datagen_info.record_object_poses = False + return recorder_cfg def build_datastream( @@ -216,6 +280,7 @@ def setup_async_generation( success_term, algorithm, pause_subtask: bool, + attempt_log: AttemptLog | None = None, ) -> dict: """Build a single :class:`DataGenerator` over ``datastream`` and ``num_envs`` async tasks.""" asyncio_event_loop = asyncio.get_event_loop() @@ -238,6 +303,7 @@ def setup_async_generation( success_term=success_term, pause_subtask=pause_subtask, stats=stats, + attempt_log=attempt_log, ) ) tasks.append(task) @@ -327,7 +393,7 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else None, + recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else _generation_recorder_cfg(), env_profile=env_profile, enable_cameras=args_cli.record_images, ) @@ -347,6 +413,9 @@ def main() -> None: task_descriptor=task_descriptor, embodiment_adapter=embodiment_adapter, ) + PreStepDatagenInfoRecorder.datastream = datastream + run_id = uuid.uuid4().hex + autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) # SkillGen needs one curobo planner per env. Mimic/DexMimic take no kwargs. motion_planners: dict | None = None @@ -362,6 +431,8 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) + attempts_file = attempts_file_for(args_cli.result_file) if args_cli.result_file else None + attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( datastream=datastream, @@ -369,10 +440,12 @@ def main() -> None: success_term=success_term, algorithm=algorithm, pause_subtask=args_cli.pause_subtask, + attempt_log=attempt_log, ) data_gen_tasks = asyncio.ensure_future(asyncio.gather(*async_components["tasks"])) generation_completed = False + generation_start = time.monotonic() try: generation_completed = env_loop( env, @@ -383,6 +456,7 @@ def main() -> None: stats=async_components["stats"], data_gen_tasks=data_gen_tasks, ) + generation_end = time.monotonic() except asyncio.CancelledError: print("Async tasks cancelled.") finally: @@ -406,9 +480,28 @@ def main() -> None: if env_profile else None ), + run_metadata={ + "run_id": run_id, + "seed": generation_policy_params.seed, + "revision": code_revision(), + "inputs": { + "source": input_record(args_cli.input_file), + "embodiment": input_record(args_cli.embodiment), + "task_descriptor": input_record(args_cli.task_descriptor), + "env_profile": input_record(args_cli.env_profile), + }, + "device": args_cli.device, + **_hardware(args_cli.device), + "num_envs": args_cli.num_envs, + "setup_time_s": generation_start - PROCESS_START, + "generation_time_s": generation_end - generation_start, + "attempts_file": Path(attempts_file).name, + }, ) finally: env.close() + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) if __name__ == "__main__": diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index 19a5e3b..cfb4183 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,7 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. | +| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`.attempts.jsonl`) is written next to it. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From 96a15bde9133fcb641cde03bf28b1f57bb518f5f Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 15:36:40 -0500 Subject: [PATCH 02/24] docs: describe what generation records per step, attempt and run --- docs/pages/concepts/data_generator.rst | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index b182383..514111b 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -121,9 +121,15 @@ report progress and decide when to stop: Results and Statistics ---------------------- -Three layers report what happened: - -* **Per attempt** — ``GenerationResult``: ``success`` and ``initial_state``. +Four layers report what happened: + +* **Per step** — every exported episode records ``step_success``, whether the task's success term + held after each step. Generation also records the end-effector pose and its commanded target under + ``obs/datagen_info``; object poses are recorded there only during annotation, for the source pool. +* **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or + ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With + ``--result_file``, each completed attempt is also written as one line of + ``.attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -141,6 +147,20 @@ Three layers report what happened: "name": "franka_bin_stack", "path": "autodata_examples/env_profiles/franka_bin_stack.yaml", "planner": "franka_stack_cube_bin" + }, + "run": { + "run_id": "3f2a9c...", + "seed": 1, + "revision": "a058fd8...", + "inputs": { + "source": {"path": "datasets/source.hdf5", "sha256": "9b1c..."}, + "embodiment": {"path": "autodata_examples/embodiments/franka_ik_rel.yaml", "sha256": "4e07..."}, + "task_descriptor": {"path": "autodata_examples/tasks/franka_cube_stack.yaml", "sha256": "c2d8..."}, + "env_profile": null + }, + "device": "cuda:0", "gpu": "...", "num_gpus": 1, "cpu": "...", "num_envs": 1, + "setup_time_s": 22.3, "generation_time_s": 228.5, + "attempts_file": "generation_result.attempts.jsonl" } } @@ -148,6 +168,12 @@ Three layers report what happened: applied — the generated dataset itself records only the base env id, so the sidecar is what documents that the scene was modified. +Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` +entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported +episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, +and the embodiment's optional datasheet ``hardware_limits``. The ``run_id`` ties the dataset to its +result file and attempt log. + The GenerationAlgorithm Plug-In ------------------------------- @@ -176,6 +202,9 @@ follow. Five class attributes declare how the algorithm differs from a vanilla M The shipped algorithms — ``mimicgen``, ``dexmimicgen``, ``skillgen`` — are described in :doc:`algorithms`. +An algorithm can also override ``attempt_extras()`` to report facts about each attempt in its +attempt-log line; SkillGen reports its motion-planning time as ``planning_time_s``. + The behavioral hook: ``plan_subtask_trajectory`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 5f3264bf2c967be453a9015be1d0d40511c27847 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 14:31:50 -0500 Subject: [PATCH 03/24] feat: add the autodata_metrics registry and the run report Metrics are plain functions registered with @metric. Each declares the evidence it requires (an HDF5 path, an env_args, run or attempt-log field, or the source dataset); without it the report says which key is missing instead of computing anything. A metric that raises is reported as an error without hiding the others. The report refuses a dataset and run record whose run ids or exported counts differ. validate_dataset.py --report computes every registered metric, --metrics loads more from a module, and --gate fails the command when a gated metric is out of bounds, not applicable or at another version. Built-in run metrics: generation success rate (stop-corrected, Wilson interval), demos per GPU-hour, time to dataset, annotation pass rate, and the failure-share and planning-time diagnostics. --- autodata_metrics/__init__.py | 13 ++ autodata_metrics/evidence.py | 94 ++++++++++ autodata_metrics/registry.py | 104 +++++++++++ autodata_metrics/report.py | 167 ++++++++++++++++++ autodata_metrics/viability.py | 160 +++++++++++++++++ .../e2e/test_mimicgen_data_generation.py | 16 +- .../e2e/test_skillgen_bin_data_generation.py | 16 +- autodata_tests/metrics/__init__.py | 2 + autodata_tests/metrics/test_report.py | 89 ++++++++++ autodata_tests/metrics/test_viability.py | 28 +++ autodata_tests/utils/utils.py | 17 ++ scripts/validate_dataset.py | 39 +++- setup.py | 1 + 13 files changed, 742 insertions(+), 4 deletions(-) create mode 100644 autodata_metrics/__init__.py create mode 100644 autodata_metrics/evidence.py create mode 100644 autodata_metrics/registry.py create mode 100644 autodata_metrics/report.py create mode 100644 autodata_metrics/viability.py create mode 100644 autodata_tests/metrics/__init__.py create mode 100644 autodata_tests/metrics/test_report.py create mode 100644 autodata_tests/metrics/test_viability.py diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py new file mode 100644 index 0000000..c9d8546 --- /dev/null +++ b/autodata_metrics/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Offline metrics of AutoData datasets and generation runs. + +Producers record evidence; :func:`build_report` computes every registered metric from it without a simulator. +Importing this package registers the built-in metrics. +""" + +from autodata_metrics import viability +from autodata_metrics.evidence import Episode, Evidence +from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable, Value, metric +from autodata_metrics.report import build_report, format_report, gate_violations diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py new file mode 100644 index 0000000..0cb1ed9 --- /dev/null +++ b/autodata_metrics/evidence.py @@ -0,0 +1,94 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What a metric reads: a dataset's episodes and ``env_args``, and its generation run's record and attempt log.""" + +from __future__ import annotations + +import h5py +import json +import numpy as np +from collections.abc import Mapping +from typing import Any + +_MISSING = object() + + +def lookup(record: Any, dotted_key: str) -> Any: + """``record[a][b]`` for ``dotted_key`` ``"a.b"``, or a sentinel that :func:`is_present` rejects.""" + for part in dotted_key.split("."): + if not isinstance(record, Mapping) or part not in record: + return _MISSING + record = record[part] + return record + + +def is_present(value: Any) -> bool: + """Whether a :func:`lookup` found its key.""" + return value is not _MISSING + + +class Episode: + """One episode of a dataset; arrays are read when accessed.""" + + def __init__(self, group: h5py.Group) -> None: + self.name = group.name.rsplit("/", 1)[-1] + self._group = group + + def __contains__(self, key: str) -> bool: + return key in self._group + + def __getitem__(self, key: str) -> np.ndarray: + return self._group[key][()] + + +class Evidence: + """A dataset, and optionally the generation run that produced it, as metrics read them. + + Attributes: + episodes: The dataset's episodes. + env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer. + result: Generation result, or None for a dataset without a run record or an interrupted run. + run: The result's ``run`` block, or empty. + attempts: Attempt-log lines, or None. + source: Evidence of the source dataset the run started from, or None. + """ + + def __init__( + self, + h5_file: h5py.File, + result: Mapping[str, Any] | None = None, + attempts: list[dict[str, Any]] | None = None, + source: Evidence | None = None, + ) -> None: + data = h5_file["data"] + raw_env_args = data.attrs.get("env_args", "{}") + self.env_args: dict[str, Any] = json.loads( + raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args + ) + self.episodes = [Episode(group) for group in data.values()] + self.result = result + self.run: Mapping[str, Any] = (result or {}).get("run", {}) + self.attempts = attempts + self.source = source + + def has(self, key: str) -> bool: + """Whether the evidence key ``key`` is present. + + A key is one of: + + - an HDF5 path present in every episode, such as ``step_success``; + - ``env_args.`` or ``run.``, a dotted path into ``env_args`` or the run block; + - ``attempts``, the attempt log, or ``attempts.``, a dotted path present in every attempt line; + - ``source``, the source dataset, or ``source.``, a key of the source dataset. + """ + head, _, rest = key.partition(".") + if head == "source": + return self.source is not None and (not rest or self.source.has(rest)) + if head == "attempts": + return bool(self.attempts) and (not rest or all(is_present(lookup(line, rest)) for line in self.attempts)) + if head == "env_args": + return is_present(lookup(self.env_args, rest)) + if head == "run": + return is_present(lookup(self.run, rest)) + return bool(self.episodes) and all(key in episode for episode in self.episodes) diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py new file mode 100644 index 0000000..c043c6f --- /dev/null +++ b/autodata_metrics/registry.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Registry of dataset and run metrics. + +A metric is a function of :class:`~autodata_metrics.evidence.Evidence` registered with :func:`metric`. It declares +the evidence it ``requires``; the runner reports it as not applicable, naming the missing key, instead of calling it +without that evidence. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from autodata_metrics.evidence import Evidence + +AREAS = ("validity", "diversity", "viability", "diagnostics") + + +class NotApplicable(Exception): + """Raised by a metric whose evidence is present but does not let it apply.""" + + +@dataclass(frozen=True) +class Value: + """What a metric computed. + + Attributes: + value: A number, or named numbers such as percentiles. + n: Sample count the value rests on. + details: Context a reader needs to interpret the value. + """ + + value: float | dict[str, float] + n: int + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MetricSpec: + """A registered metric and how to report it.""" + + name: str + compute: Callable[[Evidence], Value] + area: str + signal: str + scope: Literal["dataset", "run"] + unit: str + better: Literal["higher", "lower"] | None + requires: tuple[str, ...] + primary: str | None + version: int + + +REGISTERED_METRICS: dict[str, MetricSpec] = {} + + +def metric( + name: str, + *, + area: str, + signal: str, + scope: Literal["dataset", "run"], + unit: str, + better: Literal["higher", "lower"] | None = None, + requires: tuple[str, ...] | list[str] = (), + primary: str | None = None, + version: int = 1, +) -> Callable[[Callable[[Evidence], Value]], Callable[[Evidence], Value]]: + """Register the decorated function as the metric ``name``. + + Args: + name: Unique metric name, used in reports and gates. + area: One of :data:`AREAS`. + signal: What the metric measures within its area, e.g. ``"task correctness"``. + scope: ``"dataset"`` metrics read episodes; ``"run"`` metrics read attempts and the run record. + unit: Unit of the value. + better: Which direction is better, if either. + requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`. + primary: For a dict value, the entry a gate checks by default. + version: Raise it whenever the definition changes, so gates pinned to the old one fail. + """ + assert area in AREAS, f"Metric {name!r} has area {area!r}; expected one of {AREAS}." + + def register(compute: Callable[[Evidence], Value]) -> Callable[[Evidence], Value]: + assert name not in REGISTERED_METRICS, f"Metric {name!r} is already registered." + REGISTERED_METRICS[name] = MetricSpec( + name=name, + compute=compute, + area=area, + signal=signal, + scope=scope, + unit=unit, + better=better, + requires=tuple(requires), + primary=primary, + version=version, + ) + return compute + + return register diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py new file mode 100644 index 0000000..efaf837 --- /dev/null +++ b/autodata_metrics/report.py @@ -0,0 +1,167 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the registered metrics of a dataset and its run, render them, and check them against a gate.""" + +from __future__ import annotations + +import contextlib +import h5py +import json +import yaml +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from autodata_metrics.evidence import Evidence +from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable +from autodata_utils.generation_result import attempts_file_for, code_revision, input_record + +SCHEMA_VERSION = 1 + + +def read_run(result_file: str) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None]: + """The generation result and its attempt log, each None when its file is missing. + + An interrupted run leaves an attempt log without a result. + """ + result_path = Path(result_file) + attempts_path = Path(attempts_file_for(result_file)) + result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None + attempts = None + if attempts_path.exists(): + lines = attempts_path.read_text(encoding="utf-8").splitlines() + attempts = [json.loads(line) for line in lines if line.strip()] + return result, attempts + + +def check_same_run(evidence: Evidence) -> list[str]: + """Refuse a dataset and run record that come from different runs; return warnings for unverifiable data. + + Run ids alone are not enough: merging datasets keeps the first file's ``env_args``, so the exported episode + count must match too. + """ + num_episodes = len(evidence.episodes) + if evidence.result is not None: + assert ( + num_episodes == evidence.result["num_success"] + ), f"The dataset holds {num_episodes} episodes but the run counts {evidence.result['num_success']} successes." + entry = evidence.env_args.get("autodata") + if entry is None: + return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] + assert entry["exported_episodes"] == num_episodes, ( + f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" + " it was merged or edited after export." + ) + run_ids = {entry["run_id"]} | {line["run_id"] for line in evidence.attempts or []} + if "run_id" in evidence.run: + run_ids.add(evidence.run["run_id"]) + assert len(run_ids) == 1, f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}" + return [] + + +def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: + """One report row: the metric's value, or why it is not applicable or failed.""" + row = { + "area": spec.area, + "signal": spec.signal, + "scope": spec.scope, + "unit": spec.unit, + "better": spec.better, + "primary": spec.primary, + "version": spec.version, + } + missing = next((key for key in spec.requires if not evidence.has(key)), None) + if missing is not None: + return {**row, "status": "not_applicable", "reason": f"missing {missing}"} + try: + value = spec.compute(evidence) + except NotApplicable as exc: + return {**row, "status": "not_applicable", "reason": str(exc)} + except Exception as exc: # A failing metric must not sink the report of every other metric. + return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} + return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + + +def build_report( + dataset: str, + result_file: str | None = None, + source: str | None = None, + metrics: Mapping[str, MetricSpec] | None = None, +) -> dict[str, Any]: + """Compute every metric for a dataset and, when ``result_file`` is given, its generation run. + + Args: + dataset: HDF5 dataset. + result_file: Generation result (``generate_dataset.py --result_file``); its attempt log is read beside it. + source: Source dataset the run started from. + metrics: Metrics to compute; defaults to every registered metric. + """ + result, attempts = read_run(result_file) if result_file else (None, None) + with contextlib.ExitStack() as stack: + source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None + evidence = Evidence(stack.enter_context(h5py.File(dataset, "r")), result, attempts, source_evidence) + warnings = check_same_run(evidence) + rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} + run_id = evidence.env_args.get("autodata", {}).get("run_id") + num_episodes = len(evidence.episodes) + return { + "schema_version": SCHEMA_VERSION, + "inputs": { + "dataset": input_record(dataset), + "result_file": result_file, + "source": input_record(source), + "run_id": run_id, + "revision": code_revision(), + }, + "num_episodes": num_episodes, + "warnings": warnings, + "metrics": rows, + } + + +def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: + """Bounds in ``gate_file`` that the report violates. + + The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the + metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or + at another version is a violation. + """ + violations = [] + for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): + row = report["metrics"].get(name) + if row is None or row["status"] != "ok": + violations.append(f"{name}: {'not reported' if row is None else row['status']}") + continue + if "version" in bound and bound["version"] != row["version"]: + violations.append(f"{name}: version {row['version']}, gate written for version {bound['version']}") + continue + key = bound.get("key", row["primary"]) + value = row["value"][key] if key else row["value"] + assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if "min" in bound and value < bound["min"]: + violations.append(f"{name}: {value:.4g} is below {bound['min']}") + if "max" in bound and value > bound["max"]: + violations.append(f"{name}: {value:.4g} is above {bound['max']}") + return violations + + +def _format_value(value: Any) -> str: + if isinstance(value, Mapping): + return ", ".join(f"{key}={_format_value(item)}" for key, item in value.items()) or "none" + return f"{value:.3g}" if isinstance(value, float) else str(value) + + +def format_report(report: Mapping[str, Any]) -> str: + """The report as an aligned text table, grouped by area.""" + rows = [("Metric", "Area", "Value", "Unit", "n")] + for name, row in sorted(report["metrics"].items(), key=lambda item: (AREAS.index(item[1]["area"]), item[0])): + if row["status"] == "ok": + rows.append((name, row["area"], _format_value(row["value"]), row["unit"], str(row["n"]))) + else: + rows.append((name, row["area"], f"{row['status']}: {row['reason']}", "", "")) + widths = [max(len(row[column]) for row in rows) for column in range(len(rows[0]))] + lines = [" ".join(cell.ljust(width) for cell, width in zip(row, widths)).rstrip() for row in rows] + lines.insert(1, " ".join("-" * width for width in widths)) + lines += [f"warning: {warning}" for warning in report["warnings"]] + return "\n".join(lines) diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py new file mode 100644 index 0000000..06446f5 --- /dev/null +++ b/autodata_metrics/viability.py @@ -0,0 +1,160 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Viability metrics: what producing the dataset cost, and diagnostics of why attempts failed.""" + +from __future__ import annotations + +import math +import numpy as np +from collections import Counter +from typing import Any + +from autodata_metrics.evidence import Evidence +from autodata_metrics.registry import NotApplicable, Value, metric + +Z_95 = 1.959963984540054 +MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION = 3 +"""Below this many logged attempts in some env, the stop correction is itself biased.""" + + +def wilson_interval(successes: int, trials: int, z: float = Z_95) -> tuple[float, float]: + """Wilson score interval for a binomial proportion.""" + assert trials > 0, "trials must be positive" + p = successes / trials + denominator = 1 + z**2 / trials + center = (p + z**2 / (2 * trials)) / denominator + half_width = z * math.sqrt(p * (1 - p) / trials + z**2 / (4 * trials**2)) / denominator + return max(0.0, center - half_width), min(1.0, center + half_width) + + +def attempts_before_stop(attempts: list[dict[str, Any]], num_envs: int) -> list[dict[str, Any]] | None: + """Attempts that started before the first env completed its last logged attempt. + + Attempts running when generation stops are cancelled unlogged, and long attempts are the likelier to be running. + Each env's cancelled attempt started after that env's last completion, so this start-time subset was fully + logged. None when some env logged fewer than :data:`MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION` attempts. + """ + last_end: dict[int, float] = {} + logged: Counter[int] = Counter() + for attempt in attempts: + last_end[attempt["env_id"]] = max(last_end.get(attempt["env_id"], 0.0), attempt["end_time_s"]) + logged[attempt["env_id"]] += 1 + if len(logged) < num_envs or min(logged.values()) < MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION: + return None + cutoff = min(last_end.values()) + return [attempt for attempt in attempts if attempt["start_time_s"] < cutoff] + + +def _counted_attempts(ev: Evidence) -> tuple[list[dict[str, Any]], bool]: + """The stop-corrected attempts when the log allows the correction, else all attempts; and which one it is.""" + corrected = attempts_before_stop(ev.attempts, ev.run["num_envs"]) if "num_envs" in ev.run else None + return (corrected, True) if corrected is not None else (ev.attempts, False) + + +@metric( + "generation_success_rate", + area="viability", + signal="generation performance", + scope="run", + unit="fraction", + better="higher", + requires=["attempts"], +) +def generation_success_rate(ev: Evidence) -> Value: + attempts, corrected = _counted_attempts(ev) + successes = sum(attempt["success"] for attempt in attempts) + low, high = wilson_interval(successes, len(attempts)) + raw_rate = sum(attempt["success"] for attempt in ev.attempts) / len(ev.attempts) + details = {"interval_95": [low, high], "raw_rate": raw_rate, "stop_corrected": corrected} + return Value(successes / len(attempts), n=len(attempts), details=details) + + +@metric( + "demos_per_gpu_hour", + area="viability", + signal="generation performance", + scope="run", + unit="demos/GPU-h", + better="higher", + requires=["run.generation_time_s", "run.num_gpus", "env_args.autodata.exported_episodes"], +) +def demos_per_gpu_hour(ev: Evidence) -> Value: + if ev.run["num_gpus"] == 0: + raise NotApplicable("the run used no GPU") + gpu_hours = ev.run["generation_time_s"] / 3600 * ev.run["num_gpus"] + exported = ev.env_args["autodata"]["exported_episodes"] + details = {key: ev.run.get(key) for key in ("gpu", "cpu", "device", "num_envs")} + return Value(exported / gpu_hours, n=exported, details=details) + + +@metric( + "time_to_dataset", + area="viability", + signal="time to dataset", + scope="run", + unit="s", + better="lower", + requires=["run.setup_time_s", "run.generation_time_s"], +) +def time_to_dataset(ev: Evidence) -> Value: + annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} + parts = { + "annotation_s": annotation.get("time_s"), + "setup_s": ev.run["setup_time_s"], + "generation_s": ev.run["generation_time_s"], + } + details = {**parts, "note": "automated steps only; manual recording and annotation time is not measured"} + return Value(sum(seconds for seconds in parts.values() if seconds is not None), n=1, details=details) + + +@metric( + "annotation_pass_rate", + area="viability", + signal="time to dataset", + scope="run", + unit="fraction", + better="higher", + requires=["source.env_args.autodata.annotation"], +) +def annotation_pass_rate(ev: Evidence) -> Value: + annotation = ev.source.env_args["autodata"]["annotation"] + if annotation["processed_episodes"] == 0: + raise NotApplicable("annotation processed no episodes") + rate = annotation["annotated_episodes"] / annotation["processed_episodes"] + return Value(rate, n=annotation["processed_episodes"], details={"input_episodes": annotation["input_episodes"]}) + + +@metric( + "failure_share", + area="diagnostics", + signal="failure causes", + scope="run", + unit="fraction of failures", + requires=["attempts"], +) +def failure_share(ev: Evidence) -> Value: + attempts, corrected = _counted_attempts(ev) + causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) + total = sum(causes.values()) + return Value( + {cause: count / total for cause, count in causes.items()}, n=total, details={"stop_corrected": corrected} + ) + + +@metric( + "planning_time", + area="diagnostics", + signal="generation performance", + scope="run", + unit="s per attempt", + better="lower", + primary="p95", + requires=["attempts.extras.planning_time_s"], +) +def planning_time(ev: Evidence) -> Value: + times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) + p50, p95 = np.percentile(times, [50, 95]) + generation_time = ev.run.get("generation_time_s") + share = float(times.sum() / generation_time) if generation_time else None + return Value({"p50": float(p50), "p95": float(p95)}, n=len(times), details={"share_of_generation_time": share}) diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index a97271e..b43439d 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -8,10 +8,12 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset +from autodata_tests.utils.utils import assert_ok_metrics, assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 +# Metrics a MimicGen run reported with its result file must compute. +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share"} def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: @@ -51,6 +53,18 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) assert_recorded_run(output_file, result_file) + report_file = os.path.join(temp_dir, "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + output_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) + assert_ok_metrics(report_file, RUN_METRICS) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 90b5a17..24814a7 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -17,10 +17,12 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset +from autodata_tests.utils.utils import assert_ok_metrics, assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 +# Metrics a SkillGen run reported with its result file must compute; planning time is SkillGen's own. +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share", "planning_time"} def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: @@ -65,6 +67,18 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: attempts = assert_recorded_run(output_file, result_file) # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) + report_file = os.path.join(temp_dir, "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + output_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) + assert_ok_metrics(report_file, RUN_METRICS) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/metrics/__init__.py b/autodata_tests/metrics/__init__.py new file mode 100644 index 0000000..57c822a --- /dev/null +++ b/autodata_tests/metrics/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py new file mode 100644 index 0000000..f87bf11 --- /dev/null +++ b/autodata_tests/metrics/test_report.py @@ -0,0 +1,89 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests of how the report runs metrics, refuses mismatched runs and gates values.""" + +import h5py +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from autodata_metrics import MetricSpec, Value, build_report, gate_violations + + +def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: + return MetricSpec( + name=name, + compute=compute, + area="viability", + signal="test", + scope="run", + unit="fraction", + better="higher", + requires=requires, + primary=None, + version=1, + ) + + +def _dataset(path: Path, num_episodes: int = 1, entry: dict | None = None) -> str: + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", **({"autodata": entry} if entry else {})}) + for index in range(num_episodes): + data.create_group(f"demo_{index}").create_dataset("actions", data=[[0.0, 1.0]]) + return str(path) + + +def _run(tmp_path: Path, run_id: str, num_success: int = 1) -> str: + result_file = tmp_path / "result.json" + result_file.write_text(json.dumps({"num_success": num_success, "run": {"run_id": run_id}})) + (tmp_path / "result.attempts.jsonl").write_text(json.dumps({"run_id": run_id, "success": True}) + "\n") + return str(result_file) + + +def test_missing_evidence_is_not_applicable_and_names_the_key(tmp_path): + report = build_report( + _dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(1.0, n=1), ("step_success",))} + ) + + assert report["metrics"]["m"]["status"] == "not_applicable" + assert report["metrics"]["m"]["reason"] == "missing step_success" + + +def test_failing_metric_is_an_error_and_the_rest_are_still_computed(tmp_path): + metrics = {"broken": _spec("broken", lambda ev: 1 / 0), "fine": _spec("fine", lambda ev: Value(0.5, n=2))} + + rows = build_report(_dataset(tmp_path / "d.hdf5"), metrics=metrics)["metrics"] + + assert rows["broken"]["status"] == "error" and "ZeroDivisionError" in rows["broken"]["reason"] + assert rows["fine"]["status"] == "ok" and rows["fine"]["value"] == 0.5 + + +def test_report_refuses_data_from_another_run_or_merged_data(tmp_path): + entry = {"run_id": "run-a", "exported_episodes": 1} + with pytest.raises(AssertionError, match="different runs"): + build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=_run(tmp_path, "run-b"), metrics={}) + with pytest.raises(AssertionError, match="merged or edited"): + build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + + +def test_gate_fails_on_bound_inapplicable_metric_and_outdated_version(tmp_path): + row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} + report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}}} + gate = tmp_path / "gate.yaml" + gate.write_text("low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\n") + + violations = gate_violations(report, str(gate)) + + assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent"] + + +def test_metrics_import_without_torch_or_isaaclab(): + code = "import sys, autodata_metrics; print(sorted({'torch', 'isaaclab', 'autodata_core'} & set(sys.modules)))" + output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout + + assert output.strip() == "[]" diff --git a/autodata_tests/metrics/test_viability.py b/autodata_tests/metrics/test_viability.py new file mode 100644 index 0000000..ecaa78e --- /dev/null +++ b/autodata_tests/metrics/test_viability.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known-answer tests of the viability statistics.""" + +import pytest + +from autodata_metrics.viability import attempts_before_stop, wilson_interval + + +def test_wilson_interval_matches_known_values(): + assert wilson_interval(0, 10) == pytest.approx((0.0, 0.2775), abs=1e-4) + assert wilson_interval(10, 10) == pytest.approx((0.7225, 1.0), abs=1e-4) + + +def _attempt(env_id: int, success: bool, start: float, duration: float) -> dict: + return {"env_id": env_id, "success": success, "start_time_s": start, "end_time_s": start + duration} + + +def test_attempts_before_stop_keeps_attempts_started_before_the_first_env_finished(): + long_successes = [_attempt(0, True, 4.0 * k, 4.0) for k in range(3)] # env 0 last finishes at 12 s + short_failures = [_attempt(1, False, 1.6 * k, 1.6) for k in range(10)] # one runs from 11.2 to 12.8 s + + kept = attempts_before_stop(long_successes + short_failures, num_envs=2) + + assert len(kept) == 11 and sum(attempt["success"] for attempt in kept) == 3 + assert attempts_before_stop(long_successes[:2] + short_failures, num_envs=2) is None + assert attempts_before_stop(short_failures, num_envs=2) is None diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index dd528a3..287137c 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -82,3 +82,20 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts + + +def assert_ok_metrics(report_file: str, expected_ok: set[str]) -> dict: + """Assert exactly ``expected_ok`` metrics computed a value; any other metric must say why it did not. + + A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which + this catches. + + Returns: + The report's metric rows. + """ + + with open(report_file, encoding="utf-8") as report_handle: + rows = json.load(report_handle)["metrics"] + ok = {name for name, row in rows.items() if row["status"] == "ok"} + assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" + return rows diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index a53db2d..a341278 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -9,18 +9,22 @@ python scripts/validate_dataset.py [ ...] python scripts/validate_dataset.py directory/*.hdf5 + python scripts/validate_dataset.py --report --result_file [--gate ] -The command exits nonzero if any supplied file is invalid. +The command exits nonzero if any supplied file is invalid or a gated metric violates its bound. """ from __future__ import annotations import argparse import h5py +import importlib import json import os import sys +from autodata_metrics import build_report, format_report, gate_violations + REQUIRED_DEMO_FIELDS = { "actions": h5py.Dataset, "initial_state": h5py.Group, @@ -163,7 +167,17 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Validate AutoData demonstration HDF5 datasets.") parser.add_argument("files", nargs="+", help="HDF5 dataset file(s) to validate.") + parser.add_argument("--report", action="store_true", help="Also report the metrics of a single dataset.") + parser.add_argument("--result_file", help="Generation result (generate_dataset.py --result_file) of the dataset.") + parser.add_argument("--source", help="Source dataset the generation started from.") + parser.add_argument( + "--metrics", action="append", default=[], help="Module registering more metrics; may be repeated." + ) + parser.add_argument("--gate", help="YAML of metric bounds; a violated bound makes the command fail.") + parser.add_argument("--report_file", help="Also write the report as JSON to this file.") args = parser.parse_args(argv) + if args.report and len(args.files) != 1: + parser.error("--report takes exactly one dataset") results = [validate_file(path) for path in args.files] print_results(results) @@ -173,7 +187,28 @@ def main(argv: list[str] | None = None) -> int: print() print(f"All {total} file(s) valid." if num_invalid == 0 else f"{num_invalid} of {total} file(s) invalid.") - return 1 if num_invalid else 0 + # A run whose every attempt failed leaves an empty dataset; its run metrics are still worth reporting. + violations = [] + if args.report and results[0].num_episodes is not None: + for module in args.metrics: + importlib.import_module(module) + report = build_report(args.files[0], result_file=args.result_file, source=args.source) + print() + print(format_report(report)) + if args.report_file: + with open(args.report_file, "w", encoding="utf-8") as report_handle: + json.dump(report, report_handle, indent=2, default=_json_value) + report_handle.write("\n") + if args.gate: + violations = gate_violations(report, args.gate) + print("\nGate passed." if not violations else "\nGate failed:\n " + "\n ".join(violations)) + + return 1 if num_invalid or violations else 0 + + +def _json_value(value: object) -> object: + """JSON form of the numpy scalars and arrays that metric values may hold.""" + return value.tolist() if hasattr(value, "tolist") else str(value) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 4c412ae..35b3a9f 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ include=[ "autodata_core*", "autodata_interfaces*", + "autodata_metrics*", "autodata_utils*", "autodata_examples*", ] From 2a5d4957fc5a07726d95b10327c003b679e6a0e0 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 15:38:38 -0500 Subject: [PATCH 04/24] docs: add the metrics reference page with a table generated from the metric declarations --- autodata_metrics/viability.py | 6 +++ docs/_ext/autodata_doc_tools.py | 35 ++++++++++++ docs/index.rst | 1 + docs/pages/references/metrics.rst | 88 +++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 docs/pages/references/metrics.rst diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 06446f5..62e91e9 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -62,6 +62,7 @@ def _counted_attempts(ev: Evidence) -> tuple[list[dict[str, Any]], bool]: requires=["attempts"], ) def generation_success_rate(ev: Evidence) -> Value: + """Successful attempts over attempts, stop-corrected when the attempt log allows, with a Wilson 95% interval.""" attempts, corrected = _counted_attempts(ev) successes = sum(attempt["success"] for attempt in attempts) low, high = wilson_interval(successes, len(attempts)) @@ -80,6 +81,7 @@ def generation_success_rate(ev: Evidence) -> Value: requires=["run.generation_time_s", "run.num_gpus", "env_args.autodata.exported_episodes"], ) def demos_per_gpu_hour(ev: Evidence) -> Value: + """Exported demonstrations per GPU-hour of generation time.""" if ev.run["num_gpus"] == 0: raise NotApplicable("the run used no GPU") gpu_hours = ev.run["generation_time_s"] / 3600 * ev.run["num_gpus"] @@ -98,6 +100,7 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: requires=["run.setup_time_s", "run.generation_time_s"], ) def time_to_dataset(ev: Evidence) -> Value: + """Wall time of the automated steps: annotation of the source, setup and generation.""" annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} parts = { "annotation_s": annotation.get("time_s"), @@ -118,6 +121,7 @@ def time_to_dataset(ev: Evidence) -> Value: requires=["source.env_args.autodata.annotation"], ) def annotation_pass_rate(ev: Evidence) -> Value: + """Share of processed recordings that annotation exported into the source dataset.""" annotation = ev.source.env_args["autodata"]["annotation"] if annotation["processed_episodes"] == 0: raise NotApplicable("annotation processed no episodes") @@ -134,6 +138,7 @@ def annotation_pass_rate(ev: Evidence) -> Value: requires=["attempts"], ) def failure_share(ev: Evidence) -> Value: + """Share of failed attempts by cause, ``planner`` or ``execution``.""" attempts, corrected = _counted_attempts(ev) causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) total = sum(causes.values()) @@ -153,6 +158,7 @@ def failure_share(ev: Evidence) -> Value: requires=["attempts.extras.planning_time_s"], ) def planning_time(ev: Evidence) -> Value: + """p50 and p95 motion-planning time per attempt, and its share of generation time.""" times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) p50, p95 = np.percentile(times, [50, 95]) generation_time = ev.run.get("generation_time_s") diff --git a/docs/_ext/autodata_doc_tools.py b/docs/_ext/autodata_doc_tools.py index feb9f27..72db1fd 100644 --- a/docs/_ext/autodata_doc_tools.py +++ b/docs/_ext/autodata_doc_tools.py @@ -8,7 +8,9 @@ in one place and can switch between internal/external targets on release. """ +import ast import re +from pathlib import Path from typing import Any from sphinx.application import Sphinx @@ -69,8 +71,41 @@ def curobo_replacer(_: Any) -> str: source[0] = re.sub(r":docker_run_curobo:", curobo_replacer, source[0]) +def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: + """Replaces ``:autodata_metrics_table:`` with a table of the built-in metrics. + + The rows are read from the ``@metric`` declarations in ``autodata_metrics/`` instead of importing them, so the + docs build needs none of the metrics' dependencies and the table cannot drift from the code. + """ + if ":autodata_metrics_table:" not in source[0]: + return + rows = [] + for path in sorted((Path(app.confdir).parent / "autodata_metrics").glob("*.py")): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if not isinstance(node, ast.FunctionDef): + continue + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "metric": + options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} + requires = ", ".join(f"``{key}``" for key in options.get("requires", ())) or "-" + definition = (ast.get_docstring(node) or "").split("\n")[0] + name = ast.literal_eval(decorator.args[0]) + rows.append((options["area"], name, options["unit"], requires, definition)) + table = [ + ".. list-table::", + " :header-rows: 1", + " :widths: 20 12 14 26 28", + "", + " * - Metric\n - Area\n - Unit\n - Requires\n - Definition", + ] + for area, name, unit, requires, definition in sorted(rows): + table.append(f" * - ``{name}``\n - {area}\n - {unit}\n - {requires}\n - {definition}") + source[0] = source[0].replace(":autodata_metrics_table:", "\n".join(table) + "\n") + + def setup(app: Sphinx) -> None: app.connect("source-read", autodata_git_clone_code_block) app.connect("source-read", autodata_code_link) app.connect("source-read", docker_run_command_replacer) + app.connect("source-read", autodata_metrics_table) app.add_config_value("autodata_docs_config", {}, "env") diff --git a/docs/index.rst b/docs/index.rst index 6d4c9bf..853caf5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -146,5 +146,6 @@ Table of Contents :maxdepth: 1 :caption: References + pages/references/metrics pages/references/troubleshooting pages/references/release_notes diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst new file mode 100644 index 0000000..a2b8073 --- /dev/null +++ b/docs/pages/references/metrics.rst @@ -0,0 +1,88 @@ +Metrics +======= + +``scripts/validate_dataset.py --report`` computes the metrics of a dataset and, given its result +file, of the generation run that produced it. Metrics are computed from recorded files and need no +simulator. + +.. code-block:: bash + + python scripts/validate_dataset.py datasets/generated.hdf5 --report \ + --result_file datasets/generated.json \ + --source datasets/annotated_datasets/dataset_franka_annotated.hdf5 \ + --report_file datasets/generated_report.json + +Each metric reports one of three things: + +- a value, with its unit and sample count; +- ``not_applicable``, with the reason, such as a missing recorded input; +- ``error``, when it fails. The other metrics are still computed. + +The report refuses a dataset and a result file from different runs, because their run ids and exported +episode counts must match. + +Gating +------ + +``--gate`` reads metric bounds from YAML and makes the command fail when one is violated: + +.. code-block:: yaml + + generation_success_rate: {min: 0.3, version: 1} + planning_time: {max: 5.0, key: p95} + +A gated metric also violates the gate if it is not applicable, fails, or reports a different +``version``. + +Built-in metrics +---------------- + +:autodata_metrics_table: + +Adding a metric +--------------- + +A metric is a function registered with ``autodata_metrics.metric``. It names the evidence it +requires. Without that evidence, the report says which key is missing and does not call the function. + +.. code-block:: python + + import numpy as np + + from autodata_metrics import Value, metric + + + @metric( + "planning_time", + area="diagnostics", + signal="generation performance", + scope="run", + unit="s per attempt", + better="lower", + primary="p95", + requires=["attempts.extras.planning_time_s"], + ) + def planning_time(ev): + """p50 and p95 motion-planning time per attempt.""" + times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) + p50, p95 = np.percentile(times, [50, 95]) + return Value({"p50": float(p50), "p95": float(p95)}, n=len(times)) + +An evidence key is one of: + +- an HDF5 path present in every episode, such as ``step_success``; +- ``env_args.`` or ``run.``; +- ``attempts`` or ``attempts.``; +- ``source`` or ``source.``. + +Built-in metrics live in ``autodata_metrics/``. Load your own with +``--metrics my_package.my_metrics``. Raise a metric's ``version`` whenever its definition changes. + +When you add an algorithm or an embodiment: + +- An algorithm reports per-attempt facts by overriding ``attempt_extras()``. Metrics that require + those facts then apply to its runs. +- An embodiment's adapter describes its pose actions through ``metric_context()``. Add datasheet + ``hardware_limits`` to its YAML to enable the joint-limit metrics. +- List the workflow's expected metrics in its end-to-end test, so a metric that stops applying fails + the test. From 5303c8fd8d6c2cfb48232214c74aaaf2836eeebd Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:12:36 -0500 Subject: [PATCH 05/24] fix: keep earlier runs' records and label only this run's datasets - Each run writes its own attempt log, ..attempts.jsonl, so rerunning with the same --result_file keeps earlier logs, and an interrupted rerun leaves the earlier result and its log consistent. - A _failed dataset gets this run's autodata entry only when this run kept failed episodes, so one left by an earlier run is not relabelled. - The e2e helper checks the logged successes against the result. - Docs: only generated datasets have a result file and attempt log. --- autodata_retargeting/runner.py | 2 +- autodata_tests/utils/test_dataset_metadata.py | 14 ++++++++++++++ autodata_tests/utils/test_generation_result.py | 13 ++++++++++++- autodata_tests/utils/utils.py | 3 +++ autodata_utils/dataset_metadata.py | 16 +++++++++++++--- autodata_utils/generation_result.py | 6 +++--- docs/pages/concepts/data_generator.rst | 8 ++++---- scripts/annotate_demos.py | 2 +- scripts/generate_dataset.py | 6 +++--- .../references/cli-reference.md | 2 +- 10 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 autodata_tests/utils/test_dataset_metadata.py diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index e4a05d2..cb82c9b 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -185,5 +185,5 @@ def run(args, simulation_app) -> int: dataset_file_handler.close() env.close() if autodata_entry is not None: - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy.keep_failed): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/autodata_tests/utils/test_dataset_metadata.py b/autodata_tests/utils/test_dataset_metadata.py new file mode 100644 index 0000000..99fe7cf --- /dev/null +++ b/autodata_tests/utils/test_dataset_metadata.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from autodata_utils.dataset_metadata import exported_dataset_files + + +def test_failed_file_left_by_an_earlier_run_is_not_exported_again(tmp_path: Path) -> None: + for name in ("generated.hdf5", "generated_failed.hdf5"): + (tmp_path / name).touch() + + assert exported_dataset_files(str(tmp_path), "generated", kept_failed=False) == [str(tmp_path / "generated.hdf5")] + assert len(exported_dataset_files(str(tmp_path), "generated", kept_failed=True)) == 2 diff --git a/autodata_tests/utils/test_generation_result.py b/autodata_tests/utils/test_generation_result.py index 31bb6b9..37fe93b 100644 --- a/autodata_tests/utils/test_generation_result.py +++ b/autodata_tests/utils/test_generation_result.py @@ -4,8 +4,9 @@ """Tests for structured data-generation result files.""" import json +from pathlib import Path -from autodata_utils.generation_result import write_generation_result +from autodata_utils.generation_result import AttemptLog, attempts_file_for, write_generation_result def test_write_generation_result_writes_completed_stats_atomically(tmp_path): @@ -52,3 +53,13 @@ def test_write_generation_result_records_env_profile_when_given(tmp_path): "path": "autodata_examples/env_profiles/franka_bin_stack.yaml", "planner": "franka_stack_cube_bin", } + + +def test_rerun_with_the_same_result_file_keeps_the_earlier_attempt_log(tmp_path: Path) -> None: + result_file = str(tmp_path / "result.json") + AttemptLog(attempts_file_for(result_file, "run-a"), "run-a").write({"success": True}) + + AttemptLog(attempts_file_for(result_file, "run-b"), "run-b") + + earlier = Path(attempts_file_for(result_file, "run-a")).read_text().splitlines() + assert [json.loads(line)["run_id"] for line in earlier] == ["run-a"] diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index dd528a3..c300e97 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -79,6 +79,9 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: attempts = [json.loads(line) for line in attempts_handle if line.strip()] assert entry["exported_episodes"] == result["num_success"], "The dataset and result count different successes." assert len(attempts) == result["num_attempts"], "The attempt log and result count different attempts." + assert ( + sum(attempt["success"] for attempt in attempts) == result["num_success"] + ), "The attempt log and result count different successes." run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts diff --git a/autodata_utils/dataset_metadata.py b/autodata_utils/dataset_metadata.py index 745077b..35ac6e9 100644 --- a/autodata_utils/dataset_metadata.py +++ b/autodata_utils/dataset_metadata.py @@ -14,9 +14,19 @@ AUTODATA_ENTRY_VERSION = 1 -def exported_dataset_files(output_dir: str, output_file_name: str) -> list[str]: - """The HDF5 files a recorder exported: the dataset and, when failed episodes were kept, its ``_failed`` file.""" - candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in ("", "_failed")] +def exported_dataset_files(output_dir: str, output_file_name: str, kept_failed: bool) -> list[str]: + """The HDF5 files this run's recorder exported. + + A ``_failed`` file counts only when this run kept failed episodes; otherwise one left by an earlier run would be + labelled with this run's id. + + Args: + output_dir: Export directory. + output_file_name: Dataset file name without the extension. + kept_failed: Whether the run exported failed episodes to a separate ``_failed`` file. + """ + suffixes = ("", "_failed") if kept_failed else ("",) + candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in suffixes] return [path for path in candidates if os.path.exists(path)] diff --git a/autodata_utils/generation_result.py b/autodata_utils/generation_result.py index 090d4db..fc5bc4e 100644 --- a/autodata_utils/generation_result.py +++ b/autodata_utils/generation_result.py @@ -75,9 +75,9 @@ def write_generation_result( os.unlink(temporary_file_name) -def attempts_file_for(result_file: str) -> str: - """Path of the attempt log written next to ``result_file``.""" - return str(Path(result_file).with_suffix(".attempts.jsonl")) +def attempts_file_for(result_file: str, run_id: str) -> str: + """Path of the run's attempt log, next to ``result_file``; each run writes its own, so a rerun keeps earlier logs.""" + return str(Path(result_file).with_suffix(f".{run_id}.attempts.jsonl")) class AttemptLog: diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 514111b..787d88d 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -129,7 +129,7 @@ Four layers report what happened: * **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With ``--result_file``, each completed attempt is also written as one line of - ``.attempts.jsonl``. + ``..attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -160,7 +160,7 @@ Four layers report what happened: }, "device": "cuda:0", "gpu": "...", "num_gpus": 1, "cpu": "...", "num_envs": 1, "setup_time_s": 22.3, "generation_time_s": 228.5, - "attempts_file": "generation_result.attempts.jsonl" + "attempts_file": "generation_result.3f2a9c.attempts.jsonl" } } @@ -171,8 +171,8 @@ Four layers report what happened: Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. The ``run_id`` ties the dataset to its -result file and attempt log. +and the embodiment's optional datasheet ``hardware_limits``. For a generated dataset, the ``run_id`` +ties it to its result file and attempt log. The GenerationAlgorithm Plug-In ------------------------------- diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index d27984c..95d370f 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -339,7 +339,7 @@ def main() -> int: finally: env.close() if autodata_entry is not None: - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, kept_failed=False): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 5a3bd48..19791a5 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -102,7 +102,7 @@ default=None, help=( "Optional JSON path for the final structured generation result. An attempt log with one JSON " - "line per attempt is written next to it (.attempts.jsonl)." + "line per attempt is written next to it (..attempts.jsonl)." ), ) parser.add_argument( @@ -431,7 +431,7 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) - attempts_file = attempts_file_for(args_cli.result_file) if args_cli.result_file else None + attempts_file = attempts_file_for(args_cli.result_file, run_id) if args_cli.result_file else None attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( @@ -500,7 +500,7 @@ def main() -> None: ) finally: env.close() - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy_params.keep_failed): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index cfb4183..9062e4a 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,7 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`.attempts.jsonl`) is written next to it. | +| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`..attempts.jsonl`) is written next to it. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From 9ca575be99aea366823bcf7f15ddbd94e4d63627 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:16:45 -0500 Subject: [PATCH 06/24] fix: report a run from the attempt log of the dataset's own run Each run now writes its own attempt log. The report reads the log named by the result when the result belongs to the dataset's run; otherwise it sets the result aside with a warning and reads the log named by the dataset's run id, so an interrupted rerun is still reported and runs are never mixed. --- autodata_metrics/evidence.py | 14 +++++++----- autodata_metrics/report.py | 33 +++++++++++++++++++-------- autodata_tests/metrics/test_report.py | 29 ++++++++++++++++++----- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 0cb1ed9..972b628 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -23,6 +23,12 @@ def lookup(record: Any, dotted_key: str) -> Any: return record +def read_env_args(h5_file: h5py.File) -> dict[str, Any]: + """The dataset's ``env_args``; its ``autodata`` entry was written by the producer.""" + raw_env_args = h5_file["data"].attrs.get("env_args", "{}") + return json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + + def is_present(value: Any) -> bool: """Whether a :func:`lookup` found its key.""" return value is not _MISSING @@ -61,12 +67,8 @@ def __init__( attempts: list[dict[str, Any]] | None = None, source: Evidence | None = None, ) -> None: - data = h5_file["data"] - raw_env_args = data.attrs.get("env_args", "{}") - self.env_args: dict[str, Any] = json.loads( - raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args - ) - self.episodes = [Episode(group) for group in data.values()] + self.env_args = read_env_args(h5_file) + self.episodes = [Episode(group) for group in h5_file["data"].values()] self.result = result self.run: Mapping[str, Any] = (result or {}).get("run", {}) self.attempts = attempts diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index efaf837..c11ac92 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -13,26 +13,38 @@ from pathlib import Path from typing import Any -from autodata_metrics.evidence import Evidence +from autodata_metrics.evidence import Evidence, read_env_args from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable from autodata_utils.generation_result import attempts_file_for, code_revision, input_record SCHEMA_VERSION = 1 -def read_run(result_file: str) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None]: - """The generation result and its attempt log, each None when its file is missing. +def read_run( + result_file: str, run_id: str | None +) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None, list[str]]: + """The generation result and attempt log of run ``run_id``, each None when missing, and any warnings. - An interrupted run leaves an attempt log without a result. + A rerun with the same result file leaves the earlier result in place until it completes, so a result from + another run is set aside and the run is reported from its own attempt log. """ result_path = Path(result_file) - attempts_path = Path(attempts_file_for(result_file)) result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None + warnings = [] + if result is not None and run_id is not None and result.get("run", {}).get("run_id") != run_id: + warnings.append(f"{result_file} belongs to another run, so the run is reported from its attempt log") + result = None + if result is not None and "attempts_file" in result.get("run", {}): + attempts_path = result_path.parent / result["run"]["attempts_file"] + elif run_id is not None: + attempts_path = Path(attempts_file_for(result_file, run_id)) + else: + return result, None, warnings attempts = None if attempts_path.exists(): lines = attempts_path.read_text(encoding="utf-8").splitlines() attempts = [json.loads(line) for line in lines if line.strip()] - return result, attempts + return result, attempts, warnings def check_same_run(evidence: Evidence) -> list[str]: @@ -97,13 +109,14 @@ def build_report( source: Source dataset the run started from. metrics: Metrics to compute; defaults to every registered metric. """ - result, attempts = read_run(result_file) if result_file else (None, None) with contextlib.ExitStack() as stack: + h5_file = stack.enter_context(h5py.File(dataset, "r")) + run_id = read_env_args(h5_file).get("autodata", {}).get("run_id") + result, attempts, warnings = read_run(result_file, run_id) if result_file else (None, None, []) source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None - evidence = Evidence(stack.enter_context(h5py.File(dataset, "r")), result, attempts, source_evidence) - warnings = check_same_run(evidence) + evidence = Evidence(h5_file, result, attempts, source_evidence) + warnings += check_same_run(evidence) rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} - run_id = evidence.env_args.get("autodata", {}).get("run_id") num_episodes = len(evidence.episodes) return { "schema_version": SCHEMA_VERSION, diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index f87bf11..7986217 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -38,10 +38,17 @@ def _dataset(path: Path, num_episodes: int = 1, entry: dict | None = None) -> st return str(path) -def _run(tmp_path: Path, run_id: str, num_success: int = 1) -> str: +def _log(tmp_path: Path, run_id: str, successes: list[bool]) -> None: + lines = [json.dumps({"run_id": run_id, "success": success}) for success in successes] + (tmp_path / f"result.{run_id}.attempts.jsonl").write_text("\n".join(lines) + "\n") + + +def _run(tmp_path: Path, run_id: str) -> str: + """A completed run: its result file and attempt log.""" result_file = tmp_path / "result.json" - result_file.write_text(json.dumps({"num_success": num_success, "run": {"run_id": run_id}})) - (tmp_path / "result.attempts.jsonl").write_text(json.dumps({"run_id": run_id, "success": True}) + "\n") + run = {"run_id": run_id, "attempts_file": f"result.{run_id}.attempts.jsonl"} + result_file.write_text(json.dumps({"num_success": 1, "num_attempts": 1, "run": run})) + _log(tmp_path, run_id, [True]) return str(result_file) @@ -63,10 +70,20 @@ def test_failing_metric_is_an_error_and_the_rest_are_still_computed(tmp_path): assert rows["fine"]["status"] == "ok" and rows["fine"]["value"] == 0.5 -def test_report_refuses_data_from_another_run_or_merged_data(tmp_path): +def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): + result_file = _run(tmp_path, "run-a") + _log(tmp_path, "run-b", [True, False]) + dataset = _dataset(tmp_path / "d.hdf5", entry={"run_id": "run-b", "exported_episodes": 1}) + count = {"attempts": _spec("attempts", lambda ev: Value(len(ev.attempts), n=1), ("attempts",))} + + report = build_report(dataset, result_file=result_file, metrics=count) + + assert report["metrics"]["attempts"]["value"] == 2 + assert "belongs to another run" in report["warnings"][0] + + +def test_report_refuses_merged_data(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} - with pytest.raises(AssertionError, match="different runs"): - build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=_run(tmp_path, "run-b"), metrics={}) with pytest.raises(AssertionError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) From 5948fc060560f3674a14203602098a52fd4328f9 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:33:20 -0500 Subject: [PATCH 07/24] docs: say the result file and attempt log come with --result_file --- docs/pages/concepts/data_generator.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 787d88d..6fa9454 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -171,8 +171,8 @@ Four layers report what happened: Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. For a generated dataset, the ``run_id`` -ties it to its result file and attempt log. +and the embodiment's optional datasheet ``hardware_limits``. For a dataset generated with +``--result_file``, the ``run_id`` ties it to that run's result file and attempt log. The GenerationAlgorithm Plug-In ------------------------------- From 77b511de37d24a0993432a7707738ea6fd4947c5 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 13:51:26 -0500 Subject: [PATCH 08/24] feat: make metric recording opt-in and keep algorithm diagnostics in extras - Metric recording is off by default, so datasets and result files are unchanged unless --record_metric_inputs is given to generation, annotation or retargeting. It records the per-step poses and task success, the autodata entry and, with --result_file, the run block and attempt log. - GenerationResult no longer has failure_cause; SkillGen reports planning_failed in its attempt extras, next to planning_time_s. - metric_context() docstrings say what they record; the pose-action entry's columns key is now action_slice. - Docs: the run linkage holds once the run completes. --- autodata_core/algorithms.py | 9 ++- autodata_core/data_generator.py | 8 +-- .../absolute_pose_embodiment_adapter.py | 7 +- .../bimanual_embodiment_adapter.py | 9 ++- .../embodiments/embodiment_adapter.py | 8 ++- .../single_arm_embodiment_adapter.py | 7 +- .../env/isaaclab_env_interface.py | 6 +- autodata_retargeting/runner.py | 4 +- autodata_tests/core/test_algorithms.py | 24 +++++++ autodata_tests/core/test_data_generator.py | 17 +---- autodata_tests/e2e/test_annotation.py | 1 + .../e2e/test_dexmimicgen_data_generation.py | 3 +- .../e2e/test_mimicgen_data_generation.py | 1 + .../e2e/test_skillgen_bin_data_generation.py | 1 + autodata_tests/utils/utils.py | 11 +++ docs/pages/concepts/data_generator.rst | 28 ++++---- scripts/annotate_demos.py | 22 ++++-- scripts/generate_dataset.py | 69 ++++++++++++------- scripts/retarget_dataset.py | 5 ++ .../references/cli-reference.md | 3 +- 20 files changed, 160 insertions(+), 83 deletions(-) diff --git a/autodata_core/algorithms.py b/autodata_core/algorithms.py index 8c969be..e0df4f8 100644 --- a/autodata_core/algorithms.py +++ b/autodata_core/algorithms.py @@ -200,8 +200,12 @@ def __init__(self, motion_planners: dict[int, Any]) -> None: self.motion_planners = motion_planners def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: - """Report the attempt's wall-clock motion-planning time [s] as ``planning_time_s``.""" - return {"planning_time_s": sum(state.planning_time_s for state in eef_states)} + """Report the attempt's wall-clock motion-planning time [s] and whether a motion plan failed.""" + states = list(eef_states) + return { + "planning_time_s": sum(state.planning_time_s for state in states), + "planning_failed": any(state.planning_failed for state in states), + } def plan_subtask_trajectory( self, @@ -261,6 +265,7 @@ def plan_subtask_trajectory( ) eef_state.planning_time_s += time.perf_counter() - start_time if not planning_success: + eef_state.planning_failed = True return None mp_waypoints = self._convert_planned_trajectory_to_waypoints(planner, target_passthrough_action) diff --git a/autodata_core/data_generator.py b/autodata_core/data_generator.py index 59d1e84..3f20fa5 100644 --- a/autodata_core/data_generator.py +++ b/autodata_core/data_generator.py @@ -15,7 +15,7 @@ import torch from copy import deepcopy from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any from isaaclab.managers import TerminationTermCfg @@ -52,15 +52,12 @@ class GenerationResult: Attributes: initial_state: Scene state of every env right after this env's reset. success: Whether the task succeeded at any step of the attempt. - failure_cause: ``"planner"`` when motion planning aborted the attempt, otherwise ``"execution"`` - for a failed attempt and None on success. extras: Algorithm-specific facts about the attempt, from :meth:`~autodata_core.algorithms.GenerationAlgorithm.attempt_extras`. """ initial_state: dict success: bool - failure_cause: Literal["planner", "execution"] | None = None extras: dict[str, Any] = field(default_factory=dict) @@ -88,6 +85,7 @@ class _EEFGenerationState: pending_subtask_trajectory: WaypointTrajectory | None = None is_in_motion_plan: bool = False planning_time_s: float = 0.0 + planning_failed: bool = False class DataGenerator: @@ -584,7 +582,6 @@ async def generate( return GenerationResult( initial_state=initial_state, success=False, - failure_cause="planner", extras=self.algorithm.attempt_extras(eef_states.values()), ) next_trajectory, _is_motion_plan = result @@ -637,7 +634,6 @@ async def generate( return GenerationResult( initial_state=initial_state, success=buffers.success, - failure_cause=None if buffers.success else "execution", extras=self.algorithm.attempt_extras(eef_states.values()), ) diff --git a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py index 33cc04c..08016f4 100644 --- a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py @@ -78,9 +78,12 @@ def from_dict(cls, data: dict[str, Any]) -> AbsolutePoseEmbodimentAdapter: return cls(**data) def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns of every EEF that commands a pose; they are not clipped.""" + """Each end effector with a ``pose_action_slice`` commands its target pose there: position, then quaternion. + + These entries are not clipped. + """ pose_action = { - name: {"columns": list(spec["pose_action_slice"]), "bound": None} + name: {"action_slice": list(spec["pose_action_slice"]), "bound": None} for name, spec in self.eefs.items() if spec.get("pose_action_slice") is not None } diff --git a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py index ad698e4..e948e3b 100644 --- a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py @@ -192,10 +192,13 @@ def action_dim(self) -> int: return max([self.hand_joints_slice[1], *channel_ends]) def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns of both hands; they are not clipped.""" + """``left_pose_slice`` and ``right_pose_slice`` of the action vector hold the hands' target poses. + + Each holds a position, then a quaternion. These entries are not clipped. + """ pose_action = { - self.left.name: {"columns": list(self.left_pose_slice), "bound": None}, - self.right.name: {"columns": list(self.right_pose_slice), "bound": None}, + self.left.name: {"action_slice": list(self.left_pose_slice), "bound": None}, + self.right.name: {"action_slice": list(self.right_pose_slice), "bound": None}, } return {**super().metric_context(), "pose_action": pose_action} diff --git a/autodata_interfaces/embodiments/embodiment_adapter.py b/autodata_interfaces/embodiments/embodiment_adapter.py index a219c6b..d0b9859 100644 --- a/autodata_interfaces/embodiments/embodiment_adapter.py +++ b/autodata_interfaces/embodiments/embodiment_adapter.py @@ -51,10 +51,12 @@ class EmbodimentAdapter(ABC): raw_config: dict[str, Any] | None = None def metric_context(self) -> dict[str, Any]: - """Facts about this embodiment that the dataset metrics read, stored with every dataset it records. + """Describe this embodiment for the dataset metrics; the description is saved with every dataset it records. - Subclasses add their pose-action layout. The base adds the optional ``hardware_limits`` section of the - embodiment config (datasheet joint limits, keyed by joint-name pattern). + The base returns the optional ``hardware_limits`` section of the embodiment config: datasheet joint limits + keyed by joint-name pattern. Subclasses add ``pose_action``, which says for each end effector which part of + the action vector commands its pose (``action_slice``, a half-open ``[start, end)`` index range) and the + value the adapter clips those entries to (``bound``, or None when they are not clipped). """ hardware_limits = (self.raw_config or {}).get("hardware_limits") return {"hardware_limits": hardware_limits} if hardware_limits else {} diff --git a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py index 3b25dec..6cd8df0 100644 --- a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py @@ -134,9 +134,12 @@ class DeltaPoseIKSingleArmAdapter(SingleArmEmbodimentAdapter): clip_pose_action_to_unit: bool = True def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns and their clip bound.""" + """Action entries 0-5 command the end effector's pose change: delta position, then axis-angle rotation. + + They are clipped to ±1 when ``clip_pose_action_to_unit`` is set. + """ bound = 1.0 if self.clip_pose_action_to_unit else None - pose_action = {self.eef_name: {"columns": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} + pose_action = {self.eef_name: {"action_slice": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} return {**super().metric_context(), "pose_action": pose_action} @property diff --git a/autodata_interfaces/env/isaaclab_env_interface.py b/autodata_interfaces/env/isaaclab_env_interface.py index 80754f0..9e4ebf6 100644 --- a/autodata_interfaces/env/isaaclab_env_interface.py +++ b/autodata_interfaces/env/isaaclab_env_interface.py @@ -200,6 +200,7 @@ def setup_env_config( recorder_cfg: RecorderManagerBaseCfg | None = None, env_profile: EnvironmentProfile | None = None, enable_cameras: bool = False, + record_step_success: bool = False, ) -> tuple[Any, Any]: """Configure the environment for data generation. @@ -220,6 +221,8 @@ def setup_env_config( reset-event changes) before the generation-specific adjustments below. enable_cameras: Record the task's image observations. Defaults to low-dimensional observations, excluding the cameras used for teleoperation previews. + record_step_success: Also record whether the task's success term holds after each step, as + ``step_success``, for the dataset metrics. Returns: A tuple of the environment configuration and the success termination condition. @@ -265,7 +268,8 @@ def setup_env_config( env_cfg.recorders = ActionStateRecorderManagerCfg() else: env_cfg.recorders = recorder_cfg - add_step_success_recorder(env_cfg.recorders, success_term) + if record_step_success: + add_step_success_recorder(env_cfg.recorders, success_term) env_cfg.recorders.dataset_export_dir_path = output_dir env_cfg.recorders.dataset_filename = output_file_name diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index cb82c9b..7621d1c 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -98,6 +98,7 @@ def run(args, simulation_app) -> int: num_envs=num_envs, device=args.device, generation_policy_params=generation_policy, + record_step_success=getattr(args, "record_metric_inputs", False), ) # Compensate the target env's (auto-detected) arm-action scale *in the encoded action* rather than @@ -122,7 +123,8 @@ def run(args, simulation_app) -> int: autodata_entry = None try: target_adapter.bind_env(env) - autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) + if getattr(args, "record_metric_inputs", False): + autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) env.reset() # Snapshot the target's home scene state once; each replay restores its robot from here # (deepcopy so later sim steps cannot mutate the cached tensors). diff --git a/autodata_tests/core/test_algorithms.py b/autodata_tests/core/test_algorithms.py index b3b0867..ae90f59 100644 --- a/autodata_tests/core/test_algorithms.py +++ b/autodata_tests/core/test_algorithms.py @@ -3,6 +3,9 @@ """Unit tests for :mod:`autodata_core.algorithms`.""" +from types import SimpleNamespace +from unittest.mock import Mock + import pytest from autodata_core.algorithms import ( @@ -14,6 +17,7 @@ get_algorithm, iter_algorithms, ) +from autodata_core.data_generator import _EEFGenerationState def test_registry_contents(): @@ -81,3 +85,23 @@ def test_get_algorithm_skillgen_needs_planners(): assert isinstance( get_algorithm("skillgen", motion_planners={0: object()}), SkillGen ) # base implementation must not raise + + +def test_skillgen_reports_a_failed_motion_plan(): + planner = Mock(update_world_and_plan_motion=Mock(return_value=False)) + algorithm = SkillGen(motion_planners={0: planner}) + data_generator = Mock() + data_generator.generate_eef_subtask_trajectory.return_value = [SimpleNamespace(pose=None, passthrough_action=None)] + eef_state = _EEFGenerationState() + + result = algorithm.plan_subtask_trajectory( + data_generator=data_generator, + env_id=0, + eef_name="franka", + eef_state=eef_state, + all_randomized_subtask_boundaries={}, + runtime_subtask_constraints_dict={}, + selected_src_demo_inds={}, + ) + + assert result is None and algorithm.attempt_extras([eef_state])["planning_failed"] is True diff --git a/autodata_tests/core/test_data_generator.py b/autodata_tests/core/test_data_generator.py index 6667ca1..806c0e9 100644 --- a/autodata_tests/core/test_data_generator.py +++ b/autodata_tests/core/test_data_generator.py @@ -196,21 +196,6 @@ def test_navigation_does_not_skip_playback_after_cancellation_or_while_paused(pa assert all(state.subtask_step_index == 4 for state in states.values()) -def test_planning_failure_is_attributed_to_the_planner(monkeypatch): - generator = _navigation_generator(active=False, reached=True) - generator.datastream.env = SimpleNamespace(recorder_manager=Mock()) - monkeypatch.setattr(generator.algorithm, "plan_subtask_trajectory", lambda **kwargs: None) - monkeypatch.setattr(generator, "_reset_environment_for_generation", AsyncMock(return_value=(torch.tensor([0]), {}))) - - result = asyncio.run( - generator.generate( - env_id=0, success_term=None, env_reset_queue=asyncio.Queue(), env_action_queue=asyncio.Queue() - ) - ) - - assert result.success is False and result.failure_cause == "planner" - - def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): generator = _navigation_generator(active=True, reached=False) generator.datastream.generation_policy.max_navigation_hold_steps = 3 @@ -234,7 +219,7 @@ def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): ) ) - assert result.success is False and result.failure_cause == "execution" + assert result.success is False assert execute.await_count == 3 assert recorder.set_success_to_episodes.call_args.args[1].item() is False recorder.export_episodes.assert_called_once() diff --git a/autodata_tests/e2e/test_annotation.py b/autodata_tests/e2e/test_annotation.py index bedc99c..fa93c2e 100644 --- a/autodata_tests/e2e/test_annotation.py +++ b/autodata_tests/e2e/test_annotation.py @@ -36,6 +36,7 @@ def test_automatic_annotation_from_dataset(tmp_path): str(input_file), "--output_file", str(output_file), + "--record_metric_inputs", ]) assert_valid_dataset(str(output_file), min_num_demos=1) entry = assert_recorded_episodes(str(output_file)) diff --git a/autodata_tests/e2e/test_dexmimicgen_data_generation.py b/autodata_tests/e2e/test_dexmimicgen_data_generation.py index fa73a6c..602c954 100644 --- a/autodata_tests/e2e/test_dexmimicgen_data_generation.py +++ b/autodata_tests/e2e/test_dexmimicgen_data_generation.py @@ -8,7 +8,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_no_metric_inputs, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -47,6 +47,7 @@ def _run_gr1_pick_place_dexmimicgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + assert_no_metric_inputs(output_file) def _run_g1_pick_place_dexmimicgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index a97271e..0552279 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -38,6 +38,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: output_file, "--result_file", result_file, + "--record_metric_inputs", "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 90b5a17..580d086 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -50,6 +50,7 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: output_file, "--result_file", result_file, + "--record_metric_inputs", "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index c300e97..4792836 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -85,3 +85,14 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts + + +def assert_no_metric_inputs(dataset_file: str) -> None: + """Assert a dataset recorded without ``--record_metric_inputs`` has no metric inputs, as before they existed.""" + + with h5py.File(dataset_file, "r") as h5_file: + data = h5_file["data"] + assert "autodata" not in json.loads(data.attrs["env_args"]), "The dataset has an autodata entry." + for name, episode in data.items(): + assert "step_success" not in episode, f"{name} records step_success." + assert "datagen_info" not in episode["obs"], f"{name} records datagen_info." diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 6fa9454..244451b 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -123,13 +123,13 @@ Results and Statistics Four layers report what happened: -* **Per step** — every exported episode records ``step_success``, whether the task's success term - held after each step. Generation also records the end-effector pose and its commanded target under - ``obs/datagen_info``; object poses are recorded there only during annotation, for the source pool. -* **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or - ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With - ``--result_file``, each completed attempt is also written as one line of - ``..attempts.jsonl``. +* **Per step** — annotation records ``obs/datagen_info`` (object, end-effector and commanded + target poses) for the source pool. With ``--record_metric_inputs``, every exported episode also + records ``step_success``, whether the task's success term held after each step, and generation + records the end-effector pose and its commanded target. +* **Per attempt** — ``GenerationResult``: ``success``, ``extras`` from the algorithm's + ``attempt_extras()``, and ``initial_state``. With ``--record_metric_inputs`` and ``--result_file``, + each completed attempt is also written as one line of ``..attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -168,11 +168,12 @@ Four layers report what happened: applied — the generated dataset itself records only the base env id, so the sidecar is what documents that the scene was modified. -Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` -entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported -episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. For a dataset generated with -``--result_file``, the ``run_id`` ties it to that run's result file and attempt log. +The ``run`` block is written only with ``--record_metric_inputs``. With that option, every exported +dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in +its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the +embodiment's pose-action layout, the robot's joint names with their simulator limits, and the +embodiment's optional datasheet ``hardware_limits``. For a dataset generated with ``--result_file``, +the ``run_id`` ties it to that run's attempt log and, once the run completes, its result file. The GenerationAlgorithm Plug-In ------------------------------- @@ -203,7 +204,8 @@ The shipped algorithms — ``mimicgen``, ``dexmimicgen``, ``skillgen`` — are d :doc:`algorithms`. An algorithm can also override ``attempt_extras()`` to report facts about each attempt in its -attempt-log line; SkillGen reports its motion-planning time as ``planning_time_s``. +attempt-log line. SkillGen reports its motion-planning time as ``planning_time_s`` and whether a +motion plan failed as ``planning_failed``. The behavioral hook: ``plan_subtask_trajectory`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index 95d370f..ee5434f 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -88,6 +88,11 @@ action="store_true", help="Render and record the task's image observations.", ) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help="Record what the dataset metrics read: per-step task success and run metadata in the dataset. Off by default.", +) AppLauncher.add_app_launcher_args(parser) parser.add_argument("--external_callback", type=callable_path, help="Callable registering environments after startup.") args_cli, environment_args = parser.parse_known_args() @@ -248,6 +253,7 @@ def main() -> int: generation_policy_params=generation_policy, recorder_cfg=DatagenInfoRecorderManagerCfg(), enable_cameras=args_cli.record_images, + record_step_success=args_cli.record_metric_inputs, ) # Only export episodes we explicitly mark successful (i.e. fully annotated). env_cfg.recorders.dataset_export_mode = DatasetExportMode.EXPORT_SUCCEEDED_ONLY @@ -271,7 +277,8 @@ def main() -> int: uses_start_signals=annotate_start_signals, ) PreStepDatagenInfoRecorder.datastream = _datastream - autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) + if args_cli.record_metric_inputs: + autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) env.reset() @@ -328,12 +335,13 @@ def main() -> int: f"\nExported {exported_episode_count} (out of {processed_episode_count}) annotated" f" episode{'s' if exported_episode_count != 1 else ''} to {args_cli.output_file}." ) - autodata_entry["annotation"] = { - "input_episodes": len(episode_names), - "processed_episodes": processed_episode_count, - "annotated_episodes": exported_episode_count, - "time_s": time.monotonic() - annotation_start, - } + if autodata_entry is not None: + autodata_entry["annotation"] = { + "input_episodes": len(episode_names), + "processed_episodes": processed_episode_count, + "annotated_episodes": exported_episode_count, + "time_s": time.monotonic() - annotation_start, + } print("Exiting the app.") return exported_episode_count finally: diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 19791a5..81ebb62 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -101,8 +101,16 @@ type=str, default=None, help=( - "Optional JSON path for the final structured generation result. An attempt log with one JSON " - "line per attempt is written next to it (..attempts.jsonl)." + "Optional JSON path for the final structured generation result. With --record_metric_inputs, an attempt " + "log with one JSON line per attempt is written next to it (..attempts.jsonl)." + ), +) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help=( + "Record what the dataset metrics read: per-step end-effector and target poses and task success, run " + "metadata in the dataset and, with --result_file, a run block and an attempt log. Off by default." ), ) parser.add_argument( @@ -221,7 +229,6 @@ def _attempt_record(env_id: int, result: GenerationResult, start_time: float, en return { "env_id": env_id, "success": result.success, - "failure_cause": result.failure_cause, "start_time_s": start_time, "end_time_s": end_time, "initial_object_positions": { @@ -393,7 +400,12 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else _generation_recorder_cfg(), + recorder_cfg=( + load_callable(args_cli.recorder_cfg)() + if args_cli.recorder_cfg + else _generation_recorder_cfg() if args_cli.record_metric_inputs else None + ), + record_step_success=args_cli.record_metric_inputs, env_profile=env_profile, enable_cameras=args_cli.record_images, ) @@ -415,7 +427,7 @@ def main() -> None: ) PreStepDatagenInfoRecorder.datastream = datastream run_id = uuid.uuid4().hex - autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) + autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) if args_cli.record_metric_inputs else None # SkillGen needs one curobo planner per env. Mimic/DexMimic take no kwargs. motion_planners: dict | None = None @@ -431,7 +443,7 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) - attempts_file = attempts_file_for(args_cli.result_file, run_id) if args_cli.result_file else None + attempts_file = attempts_file_for(args_cli.result_file, run_id) if autodata_entry and args_cli.result_file else None attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( @@ -480,28 +492,35 @@ def main() -> None: if env_profile else None ), - run_metadata={ - "run_id": run_id, - "seed": generation_policy_params.seed, - "revision": code_revision(), - "inputs": { - "source": input_record(args_cli.input_file), - "embodiment": input_record(args_cli.embodiment), - "task_descriptor": input_record(args_cli.task_descriptor), - "env_profile": input_record(args_cli.env_profile), - }, - "device": args_cli.device, - **_hardware(args_cli.device), - "num_envs": args_cli.num_envs, - "setup_time_s": generation_start - PROCESS_START, - "generation_time_s": generation_end - generation_start, - "attempts_file": Path(attempts_file).name, - }, + run_metadata=( + None + if autodata_entry is None + else { + "run_id": run_id, + "seed": generation_policy_params.seed, + "revision": code_revision(), + "inputs": { + "source": input_record(args_cli.input_file), + "embodiment": input_record(args_cli.embodiment), + "task_descriptor": input_record(args_cli.task_descriptor), + "env_profile": input_record(args_cli.env_profile), + }, + "device": args_cli.device, + **_hardware(args_cli.device), + "num_envs": args_cli.num_envs, + "setup_time_s": generation_start - PROCESS_START, + "generation_time_s": generation_end - generation_start, + "attempts_file": Path(attempts_file).name, + } + ), ) finally: env.close() - for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy_params.keep_failed): - merge_autodata_entry(dataset_file, autodata_entry) + if autodata_entry is not None: + for dataset_file in exported_dataset_files( + output_dir, output_file_name, generation_policy_params.keep_failed + ): + merge_autodata_entry(dataset_file, autodata_entry) if __name__ == "__main__": diff --git a/scripts/retarget_dataset.py b/scripts/retarget_dataset.py index d2906ad..af10815 100755 --- a/scripts/retarget_dataset.py +++ b/scripts/retarget_dataset.py @@ -138,6 +138,11 @@ "expected to be low." ), ) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help="Record what the dataset metrics read: per-step task success and run metadata in the dataset. Off by default.", +) AppLauncher.add_app_launcher_args(parser) args_cli = parser.parse_args() diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index 9062e4a..39d00a8 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,8 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`..attempts.jsonl`) is written next to it. | +| `--result_file ` | no | — | Where to write the generation result summary. With `--record_metric_inputs`, an attempt log (`..attempts.jsonl`) is written next to it. | +| `--record_metric_inputs` | no | off | Record what the dataset metrics read: per-step poses and task success, run metadata, and the attempt log. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From ca4a948666bca47ec86a100c08513c49ef672e74 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 13:59:23 -0500 Subject: [PATCH 09/24] fix: reject non-finite values, require --report for report options, run metrics tests in CI - The report marks a metric whose value is not a finite number as an error, and the gate counts a missing or non-numeric value as a violation. - validate_dataset.py errors when --result_file, --source, --metrics, --gate or --report_file is given without --report. - failure_share is replaced by planning_failure_rate, read from SkillGen's planning_failed attempt extra. - CI runs autodata_tests/metrics/. - Docs: metrics need --record_metric_inputs at recording time. --- .github/workflows/ci.yml | 2 +- autodata_metrics/report.py | 16 ++++++++++++++- autodata_metrics/viability.py | 18 ++++++++--------- .../e2e/test_mimicgen_data_generation.py | 2 +- .../e2e/test_skillgen_bin_data_generation.py | 10 ++++++++-- autodata_tests/metrics/test_report.py | 20 +++++++++++++++---- autodata_tests/utils/test_validate_dataset.py | 9 +++++++++ docs/pages/references/metrics.rst | 3 ++- scripts/validate_dataset.py | 10 ++++++++++ 9 files changed, 70 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5051c..20ea111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: # trials each) are heavy and run in the nightly instead. No marker filter, so # both subprocess and unit tests run. env: - TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/" + TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/ autodata_tests/metrics/" steps: - name: nvidia-smi diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index c11ac92..41d7133 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -8,6 +8,8 @@ import contextlib import h5py import json +import math +import numbers import yaml from collections.abc import Mapping from pathlib import Path @@ -92,9 +94,18 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": str(exc)} except Exception as exc: # A failing metric must not sink the report of every other metric. return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} + if not _is_finite(value.value): + return {**row, "status": "error", "reason": f"non-finite value {value.value}"} return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} +def _is_finite(value: Any) -> bool: + """Whether ``value`` is a finite number, or a mapping of finite numbers.""" + if isinstance(value, Mapping): + return all(_is_finite(item) for item in value.values()) + return isinstance(value, numbers.Real) and math.isfinite(value) + + def build_report( dataset: str, result_file: str | None = None, @@ -150,8 +161,11 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: violations.append(f"{name}: version {row['version']}, gate written for version {bound['version']}") continue key = bound.get("key", row["primary"]) - value = row["value"][key] if key else row["value"] + value = row["value"].get(key) if key else row["value"] assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if not _is_finite(value): + violations.append(f"{name}: {value!r} is not a finite number") + continue if "min" in bound and value < bound["min"]: violations.append(f"{name}: {value:.4g} is below {bound['min']}") if "max" in bound and value > bound["max"]: diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 62e91e9..a8de7a1 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -130,21 +130,19 @@ def annotation_pass_rate(ev: Evidence) -> Value: @metric( - "failure_share", + "planning_failure_rate", area="diagnostics", signal="failure causes", scope="run", - unit="fraction of failures", - requires=["attempts"], + unit="fraction of attempts", + better="lower", + requires=["attempts.extras.planning_failed"], ) -def failure_share(ev: Evidence) -> Value: - """Share of failed attempts by cause, ``planner`` or ``execution``.""" +def planning_failure_rate(ev: Evidence) -> Value: + """Share of attempts whose motion planning failed, stop-corrected when the attempt log allows.""" attempts, corrected = _counted_attempts(ev) - causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) - total = sum(causes.values()) - return Value( - {cause: count / total for cause, count in causes.items()}, n=total, details={"stop_corrected": corrected} - ) + failed = sum(attempt["extras"]["planning_failed"] for attempt in attempts) + return Value(failed / len(attempts), n=len(attempts), details={"stop_corrected": corrected}) @metric( diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index f65c7fc..08d996f 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -13,7 +13,7 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 # Metrics a MimicGen run reported with its result file must compute. -RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share"} +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset"} def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index b61db5d..b464095 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -21,8 +21,14 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 -# Metrics a SkillGen run reported with its result file must compute; planning time is SkillGen's own. -RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share", "planning_time"} +# Metrics a SkillGen run reported with its result file must compute; the planning metrics are SkillGen's own. +RUN_METRICS = { + "generation_success_rate", + "demos_per_gpu_hour", + "time_to_dataset", + "planning_time", + "planning_failure_rate", +} def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 7986217..69579ac 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -88,15 +88,27 @@ def test_report_refuses_merged_data(tmp_path): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) -def test_gate_fails_on_bound_inapplicable_metric_and_outdated_version(tmp_path): +def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} - report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}}} + missing = {"status": "ok", "value": {"p50": 1.0}, "primary": "p95", "version": 1} + report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}, "gap": missing}} gate = tmp_path / "gate.yaml" - gate.write_text("low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\n") + gate.write_text( + "low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\ngap: {max: 2}\n" + ) violations = gate_violations(report, str(gate)) - assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent"] + assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] + + +def test_non_finite_value_is_an_error_and_fails_the_gate(tmp_path): + report = build_report(_dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(float("nan"), n=1))}) + gate = tmp_path / "gate.yaml" + gate.write_text("m: {min: 0}\n") + + assert report["metrics"]["m"]["status"] == "error" + assert gate_violations(report, str(gate)) == ["m: error"] def test_metrics_import_without_torch_or_isaaclab(): diff --git a/autodata_tests/utils/test_validate_dataset.py b/autodata_tests/utils/test_validate_dataset.py index 3e1ea73..d109227 100644 --- a/autodata_tests/utils/test_validate_dataset.py +++ b/autodata_tests/utils/test_validate_dataset.py @@ -7,6 +7,8 @@ import json from pathlib import Path +import pytest + from scripts import validate_dataset @@ -45,3 +47,10 @@ def test_main_returns_nonzero_for_missing_file(tmp_path: Path) -> None: """An unreadable input path should produce a failing exit status.""" assert validate_dataset.main([str(tmp_path / "missing.hdf5")]) == 1 + + +def test_report_options_without_report_are_an_error(tmp_path: Path) -> None: + """A gate given without --report would otherwise be silently ignored.""" + + with pytest.raises(SystemExit): + validate_dataset.main([str(tmp_path / "data.hdf5"), "--gate", str(tmp_path / "gate.yaml")]) diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index a2b8073..b327d61 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -3,7 +3,8 @@ Metrics ``scripts/validate_dataset.py --report`` computes the metrics of a dataset and, given its result file, of the generation run that produced it. Metrics are computed from recorded files and need no -simulator. +simulator. Generate, annotate or retarget with ``--record_metric_inputs`` to record what they read; +without it, most metrics report that their inputs are missing. .. code-block:: bash diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index a341278..fd8aa08 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -178,6 +178,16 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.report and len(args.files) != 1: parser.error("--report takes exactly one dataset") + report_options = { + "--result_file": args.result_file, + "--source": args.source, + "--metrics": args.metrics, + "--gate": args.gate, + "--report_file": args.report_file, + } + given = [option for option, value in report_options.items() if value] + if given and not args.report: + parser.error(f"{', '.join(given)} only apply with --report") results = [validate_file(path) for path in args.files] print_results(results) From 9ea6a76ae42092d56bccacdb6648a3e4b796816b Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 16:37:03 -0500 Subject: [PATCH 10/24] fix: record metric inputs with a custom recorder and without a success term - With --record_metric_inputs, the per-step pose term is added to a custom --recorder_cfg too, unless it already records datagen info. - Without a success term, step_success is not recorded instead of failing on the first step. --- .../env/datagen_info_recorder.py | 22 +++++++++++++-- .../env/test_datagen_info_recorder.py | 28 +++++++++++++++++++ scripts/generate_dataset.py | 24 ++++++---------- 3 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 autodata_tests/interfaces/env/test_datagen_info_recorder.py diff --git a/autodata_interfaces/env/datagen_info_recorder.py b/autodata_interfaces/env/datagen_info_recorder.py index 15bb80b..81d5fa6 100644 --- a/autodata_interfaces/env/datagen_info_recorder.py +++ b/autodata_interfaces/env/datagen_info_recorder.py @@ -69,11 +69,27 @@ class DatagenInfoRecorderManagerCfg(ActionStateRecorderManagerCfg): record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() -def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg) -> None: +def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg | None) -> None: """Add the ``step_success`` term to any recorder manager configuration. + Nothing is added without a success term, since there is then no success to record. + + Args: + recorder_cfg: Recorder manager configuration the env will be built with. + success_term: The task's success term, or None when the task has none. + """ + if success_term is not None: + recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) + + +def add_pose_recorder(recorder_cfg: object) -> None: + """Add the per-step end-effector and commanded target pose term to any recorder manager configuration. + + Object poses are left out, as no metric reads them. A configuration that already has the + ``record_pre_step_datagen_info`` term, such as :class:`DatagenInfoRecorderManagerCfg`, keeps its own. + Args: recorder_cfg: Recorder manager configuration the env will be built with. - success_term: The task's success term. """ - recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) + if not hasattr(recorder_cfg, "record_pre_step_datagen_info"): + recorder_cfg.record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg(record_object_poses=False) diff --git a/autodata_tests/interfaces/env/test_datagen_info_recorder.py b/autodata_tests/interfaces/env/test_datagen_info_recorder.py new file mode 100644 index 0000000..2929133 --- /dev/null +++ b/autodata_tests/interfaces/env/test_datagen_info_recorder.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg + +from autodata_interfaces.env.datagen_info_recorder import ( + DatagenInfoRecorderManagerCfg, + add_pose_recorder, + add_step_success_recorder, +) + + +def test_metric_recorders_extend_a_custom_configuration(): + custom = ActionStateRecorderManagerCfg() + + add_pose_recorder(custom) + add_step_success_recorder(custom, success_term=None) + + assert custom.record_pre_step_datagen_info.record_object_poses is False + assert not hasattr(custom, "record_post_step_success") + + +def test_pose_recorder_keeps_an_existing_datagen_info_term(): + annotation = DatagenInfoRecorderManagerCfg() + + add_pose_recorder(annotation) + + assert annotation.record_pre_step_datagen_info.record_object_poses is True diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 81ebb62..2aa3604 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -160,6 +160,8 @@ from pathlib import Path # noqa: E402 from typing import Any # noqa: E402 +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg # noqa: E402 + from autodata_core import DataGenerator, get_algorithm # noqa: E402 from autodata_core.algorithms import REGISTERED_ALGORITHMS # noqa: E402 from autodata_core.data_generator import GenerationResult # noqa: E402 @@ -172,10 +174,7 @@ setup_env_config, setup_output_paths, ) -from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 - DatagenInfoRecorderManagerCfg, - PreStepDatagenInfoRecorder, -) +from autodata_interfaces.env.datagen_info_recorder import PreStepDatagenInfoRecorder, add_pose_recorder # noqa: E402 from autodata_interfaces.env.registration import load_callable, register_external_environment # noqa: E402 from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 from autodata_utils.generation_result import ( # noqa: E402 @@ -253,13 +252,6 @@ def _hardware(device: str) -> dict[str, str | None]: return {"gpu": gpu, "num_gpus": 1 if gpu else 0, "cpu": cpu or platform.processor() or None} -def _generation_recorder_cfg() -> DatagenInfoRecorderManagerCfg: - """Actions and states, plus the per-step poses the dataset metrics read; object poses are left out.""" - recorder_cfg = DatagenInfoRecorderManagerCfg() - recorder_cfg.record_pre_step_datagen_info.record_object_poses = False - return recorder_cfg - - def build_datastream( env: Any, input_file: str, @@ -393,6 +385,10 @@ def main() -> None: # names the motion-planner profile tuned for the resulting scene. env_profile = EnvironmentProfile.from_yaml(args_cli.env_profile) if args_cli.env_profile else None + recorder_cfg = load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else None + if args_cli.record_metric_inputs: + recorder_cfg = recorder_cfg or ActionStateRecorderManagerCfg() + add_pose_recorder(recorder_cfg) env_cfg, success_term = setup_env_config( env_name=env_name, output_dir=output_dir, @@ -400,11 +396,7 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=( - load_callable(args_cli.recorder_cfg)() - if args_cli.recorder_cfg - else _generation_recorder_cfg() if args_cli.record_metric_inputs else None - ), + recorder_cfg=recorder_cfg, record_step_success=args_cli.record_metric_inputs, env_profile=env_profile, enable_cameras=args_cli.record_images, From c632ff64b03734af43f0075dcf843c48b43de551 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 18:57:59 -0500 Subject: [PATCH 11/24] feat: add episode checks and validity metrics - Episode checks run before any metric: finite values, aligned per-step lengths, valid observed rotations, pose actions within their clip bound, and camera streams (one frame per step, no black frames, no frozen runs). An episode failing a check is counted in correctness_pass_rate or image_stream_pass_rate and left out of every dataset metric; the report lists the failures per check. - final_success_rate: share of episodes whose task success holds at their last step. - action_rotation_error: how far the commanded target rotations are from valid rotations. - A metric declared with baseline=True is also computed on the source dataset, using the run's autodata entry and control step. - Docs: the metrics page lists the checks, generated from their @check declarations. --- autodata_metrics/__init__.py | 14 +- autodata_metrics/evidence.py | 48 +++- autodata_metrics/registry.py | 48 +++- autodata_metrics/report.py | 54 ++++- autodata_metrics/validity.py | 213 ++++++++++++++++++ .../e2e/test_mimicgen_data_generation.py | 13 +- .../e2e/test_skillgen_bin_data_generation.py | 9 +- autodata_tests/metrics/test_validity.py | 112 +++++++++ docs/_ext/autodata_doc_tools.py | 37 ++- docs/pages/references/metrics.rst | 9 + 10 files changed, 536 insertions(+), 21 deletions(-) create mode 100644 autodata_metrics/validity.py create mode 100644 autodata_tests/metrics/test_validity.py diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py index c9d8546..64b9ee0 100644 --- a/autodata_metrics/__init__.py +++ b/autodata_metrics/__init__.py @@ -7,7 +7,17 @@ Importing this package registers the built-in metrics. """ -from autodata_metrics import viability +from autodata_metrics import validity, viability from autodata_metrics.evidence import Episode, Evidence -from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable, Value, metric +from autodata_metrics.registry import ( + AREAS, + REGISTERED_CHECKS, + REGISTERED_METRICS, + CheckSpec, + MetricSpec, + NotApplicable, + Value, + check, + metric, +) from autodata_metrics.report import build_report, format_report, gate_violations diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 972b628..ccea6f5 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -5,10 +5,11 @@ from __future__ import annotations +import copy import h5py import json import numpy as np -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from typing import Any _MISSING = object() @@ -47,12 +48,27 @@ def __contains__(self, key: str) -> bool: def __getitem__(self, key: str) -> np.ndarray: return self._group[key][()] + def children(self, key: str) -> list[str]: + """Names under the group ``key``, such as the end effectors under ``obs/datagen_info/eef_pose``.""" + return list(self._group[key]) + + def datasets(self) -> Iterator[tuple[str, h5py.Dataset]]: + """Every dataset of the episode with its path, unread until accessed.""" + found: list[tuple[str, h5py.Dataset]] = [] + self._group.visititems( + lambda path, item: found.append((path, item)) if isinstance(item, h5py.Dataset) else None + ) + yield from found + class Evidence: """A dataset, and optionally the generation run that produced it, as metrics read them. Attributes: - episodes: The dataset's episodes. + all_episodes: The dataset's episodes. + episodes: The episodes that passed every episode check; dataset metrics read these. + failures: Per episode check that ran, the failing episodes' names and reasons. + check_groups: Per episode check that ran, its group. env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer. result: Generation result, or None for a dataset without a run record or an interrupted run. run: The result's ``run`` block, or empty. @@ -68,7 +84,10 @@ def __init__( source: Evidence | None = None, ) -> None: self.env_args = read_env_args(h5_file) - self.episodes = [Episode(group) for group in h5_file["data"].values()] + self.all_episodes = [Episode(group) for group in h5_file["data"].values()] + self.episodes = list(self.all_episodes) + self.failures: dict[str, dict[str, str]] = {} + self.check_groups: dict[str, str] = {} self.result = result self.run: Mapping[str, Any] = (result or {}).get("run", {}) self.attempts = attempts @@ -94,3 +113,26 @@ def has(self, key: str) -> bool: if head == "run": return is_present(lookup(self.run, rest)) return bool(self.episodes) and all(key in episode for episode in self.episodes) + + @property + def entry(self) -> Mapping[str, Any]: + """The ``autodata`` entry of ``env_args``, or empty.""" + return self.env_args.get("autodata", {}) + + @property + def step_dt(self) -> float | None: + """Control step [s]: the simulation step times the decimation, or None when not recorded.""" + sim_args = self.env_args.get("sim_args") or {} + return float(sim_args["dt"]) * int(sim_args["decimation"]) if {"dt", "decimation"} <= set(sim_args) else None + + def as_baseline_for(self, run: Evidence) -> Evidence: + """This source dataset as the baseline of ``run``: its own episodes, with the run's entry and control step. + + Shipped sources predate the ``autodata`` entry, so a metric reads the robot's layout and limits from the run. + """ + baseline = copy.copy(self) + baseline.env_args = {**self.env_args, "autodata": run.entry} + if self.step_dt is None and "sim_args" in run.env_args: + baseline.env_args["sim_args"] = run.env_args["sim_args"] + baseline.result, baseline.run, baseline.attempts, baseline.source = None, {}, None, None + return baseline diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py index c043c6f..b812429 100644 --- a/autodata_metrics/registry.py +++ b/autodata_metrics/registry.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: - from autodata_metrics.evidence import Evidence + from autodata_metrics.evidence import Episode, Evidence AREAS = ("validity", "diversity", "viability", "diagnostics") @@ -47,15 +47,28 @@ class MetricSpec: compute: Callable[[Evidence], Value] area: str signal: str - scope: Literal["dataset", "run"] + scope: Literal["episode", "dataset", "run"] unit: str better: Literal["higher", "lower"] | None requires: tuple[str, ...] primary: str | None version: int + baseline: bool = False + + +@dataclass(frozen=True) +class CheckSpec: + """A registered episode check.""" + + name: str + check: Callable[[Episode, Evidence], str | None] + group: str + requires: tuple[str, ...] REGISTERED_METRICS: dict[str, MetricSpec] = {} +REGISTERED_CHECKS: dict[str, CheckSpec] = {} +CHECK_GROUPS = ("correctness", "image") def metric( @@ -63,12 +76,13 @@ def metric( *, area: str, signal: str, - scope: Literal["dataset", "run"], + scope: Literal["episode", "dataset", "run"], unit: str, better: Literal["higher", "lower"] | None = None, requires: tuple[str, ...] | list[str] = (), primary: str | None = None, version: int = 1, + baseline: bool = False, ) -> Callable[[Callable[[Evidence], Value]], Callable[[Evidence], Value]]: """Register the decorated function as the metric ``name``. @@ -76,12 +90,14 @@ def metric( name: Unique metric name, used in reports and gates. area: One of :data:`AREAS`. signal: What the metric measures within its area, e.g. ``"task correctness"``. - scope: ``"dataset"`` metrics read episodes; ``"run"`` metrics read attempts and the run record. + scope: ``"episode"`` metrics summarize the episode checks over every episode; ``"dataset"`` metrics read + the episodes that passed them; ``"run"`` metrics read attempts and the run record. unit: Unit of the value. better: Which direction is better, if either. requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`. primary: For a dict value, the entry a gate checks by default. version: Raise it whenever the definition changes, so gates pinned to the old one fail. + baseline: Also compute the metric on the source dataset, with the generated run's context. """ assert area in AREAS, f"Metric {name!r} has area {area!r}; expected one of {AREAS}." @@ -98,7 +114,31 @@ def register(compute: Callable[[Evidence], Value]) -> Callable[[Evidence], Value requires=tuple(requires), primary=primary, version=version, + baseline=baseline, ) return compute return register + + +def check( + name: str, *, group: str = "correctness", requires: tuple[str, ...] | list[str] = () +) -> Callable[[Callable[[Episode, Evidence], str | None]], Callable[[Episode, Evidence], str | None]]: + """Register the decorated function as the episode check ``name``. + + A check returns None for an episode that passes, a reason for one that fails, or raises :class:`NotApplicable` + for an episode it does not cover. Episodes that fail any check are left out of every dataset metric. + + Args: + name: Unique check name. + group: One of :data:`CHECK_GROUPS`; the pass-rate metric the check counts toward. + requires: Evidence keys the check needs; without them it does not run. + """ + assert group in CHECK_GROUPS, f"Check {name!r} has group {group!r}; expected one of {CHECK_GROUPS}." + + def register(function: Callable[[Episode, Evidence], str | None]) -> Callable[[Episode, Evidence], str | None]: + assert name not in REGISTERED_CHECKS, f"Check {name!r} is already registered." + REGISTERED_CHECKS[name] = CheckSpec(name=name, check=function, group=group, requires=tuple(requires)) + return function + + return register diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 41d7133..e962dad 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -16,7 +16,7 @@ from typing import Any from autodata_metrics.evidence import Evidence, read_env_args -from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable +from autodata_metrics.registry import AREAS, REGISTERED_CHECKS, REGISTERED_METRICS, CheckSpec, MetricSpec, NotApplicable from autodata_utils.generation_result import attempts_file_for, code_revision, input_record SCHEMA_VERSION = 1 @@ -74,7 +74,31 @@ def check_same_run(evidence: Evidence) -> list[str]: return [] -def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: +def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: + """Run the episode checks and keep only the episodes that pass all of them in ``evidence.episodes``.""" + for spec in checks.values(): + if any(not evidence.has(key) for key in spec.requires): + continue + failed: dict[str, str] = {} + applied = False + for episode in evidence.all_episodes: + try: + reason = spec.check(episode, evidence) + except NotApplicable: + continue + except Exception as exc: # A broken check fails its episodes visibly instead of sinking the report. + reason = f"{type(exc).__name__}: {exc}" + applied = True + if reason is not None: + failed[episode.name] = reason + if applied: + evidence.failures[spec.name] = failed + evidence.check_groups[spec.name] = spec.group + failing = {name for failed in evidence.failures.values() for name in failed} + evidence.episodes = [episode for episode in evidence.all_episodes if episode.name not in failing] + + +def evaluate(spec: MetricSpec, evidence: Evidence, with_baseline: bool = True) -> dict[str, Any]: """One report row: the metric's value, or why it is not applicable or failed.""" row = { "area": spec.area, @@ -85,6 +109,8 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: "primary": spec.primary, "version": spec.version, } + if spec.scope == "dataset" and not evidence.episodes: + return {**row, "status": "not_applicable", "reason": "no valid episodes"} missing = next((key for key in spec.requires if not evidence.has(key)), None) if missing is not None: return {**row, "status": "not_applicable", "reason": f"missing {missing}"} @@ -96,7 +122,10 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} if not _is_finite(value.value): return {**row, "status": "error", "reason": f"non-finite value {value.value}"} - return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + row = {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + if with_baseline and spec.baseline and evidence.source is not None: + row["baseline"] = evaluate(spec, evidence.source.as_baseline_for(evidence), with_baseline=False).get("value") + return row def _is_finite(value: Any) -> bool: @@ -111,6 +140,7 @@ def build_report( result_file: str | None = None, source: str | None = None, metrics: Mapping[str, MetricSpec] | None = None, + checks: Mapping[str, CheckSpec] | None = None, ) -> dict[str, Any]: """Compute every metric for a dataset and, when ``result_file`` is given, its generation run. @@ -119,6 +149,7 @@ def build_report( result_file: Generation result (``generate_dataset.py --result_file``); its attempt log is read beside it. source: Source dataset the run started from. metrics: Metrics to compute; defaults to every registered metric. + checks: Episode checks to run first; defaults to every registered check. """ with contextlib.ExitStack() as stack: h5_file = stack.enter_context(h5py.File(dataset, "r")) @@ -127,8 +158,16 @@ def build_report( source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) + for checked in (evidence, source_evidence): + if checked is not None: + run_checks(checked, REGISTERED_CHECKS if checks is None else checks) rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} - num_episodes = len(evidence.episodes) + num_episodes = len(evidence.all_episodes) + num_valid = len(evidence.episodes) + failures = { + name: {"failed": len(failed), "examples": dict(list(failed.items())[:3])} + for name, failed in evidence.failures.items() + } return { "schema_version": SCHEMA_VERSION, "inputs": { @@ -139,6 +178,8 @@ def build_report( "revision": code_revision(), }, "num_episodes": num_episodes, + "num_valid_episodes": num_valid, + "check_failures": failures, "warnings": warnings, "metrics": rows, } @@ -190,5 +231,10 @@ def format_report(report: Mapping[str, Any]) -> str: widths = [max(len(row[column]) for row in rows) for column in range(len(rows[0]))] lines = [" ".join(cell.ljust(width) for cell, width in zip(row, widths)).rstrip() for row in rows] lines.insert(1, " ".join("-" * width for width in widths)) + failed = [f"{name}={entry['failed']}" for name, entry in report["check_failures"].items() if entry["failed"]] + lines.append( + f"episodes: {report['num_valid_episodes']} valid of {report['num_episodes']}" + + (f"; failed checks: {', '.join(failed)}" if failed else "") + ) lines += [f"warning: {warning}" for warning in report["warnings"]] return "\n".join(lines) diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py new file mode 100644 index 0000000..1344550 --- /dev/null +++ b/autodata_metrics/validity.py @@ -0,0 +1,213 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validity metrics: whether the recorded data is well formed and the demonstrated task holds.""" + +from __future__ import annotations + +import h5py +import numpy as np + +from autodata_metrics.evidence import Episode, Evidence +from autodata_metrics.registry import NotApplicable, Value, check, metric + +ROTATION_TOLERANCE = 1e-3 +"""Largest deviation from an orthonormal rotation, or from a unit quaternion, a recorded pose may show.""" +BLACK_FRAME_MEAN = 2.0 +"""Mean pixel value, out of 255, below which a camera frame counts as black.""" +FROZEN_FRAMES = 10 +"""Identical consecutive camera frames that count as a frozen stream.""" +FRAME_CHUNK = 64 +"""Camera frames read at a time, so image checks stream instead of loading whole episodes.""" + + +def _is_image(dataset: h5py.Dataset) -> bool: + return dataset.dtype == np.uint8 and dataset.ndim == 4 + + +def _image_streams(episode: Episode) -> list[tuple[str, h5py.Dataset]]: + streams = [(path, dataset) for path, dataset in episode.datasets() if _is_image(dataset)] + if not streams: + raise NotApplicable("no camera streams") + return streams + + +@check("finite") +def finite(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode with a NaN or infinite value in any floating-point array.""" + bad = [ + path for path, dataset in episode.datasets() if dataset.dtype.kind == "f" and not np.isfinite(dataset[()]).all() + ] + return f"non-finite values in {', '.join(bad)}" if bad else None + + +PER_EPISODE_PATHS = ("initial_state", "obs/datagen_info/subtask_term_signals", "obs/datagen_info/subtask_start_signals") +"""Paths whose arrays are not one entry per step: the initial state, and annotation's subtask signals.""" + + +@check("aligned_lengths") +def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode whose per-step arrays do not all hold one entry per action; camera streams are checked apart.""" + if "actions" not in episode: + return "no actions" + num_steps = len(episode["actions"]) + bad = [ + path + for path, dataset in episode.datasets() + if not path.startswith(PER_EPISODE_PATHS) and not _is_image(dataset) and len(dataset) != num_steps + ] + return f"{', '.join(bad)} do not hold {num_steps} steps" if bad else None + + +OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") +"""Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" + + +def _rotation_errors(poses: np.ndarray) -> np.ndarray: + """Per pose, the largest entry of ``|R^T R - I|``; all-zero poses, a channel's placeholder target, are skipped.""" + rotations = poses[np.abs(poses).reshape(len(poses), -1).max(axis=1) > 0][..., :3, :3] + return np.abs(np.einsum("...ji,...jk->...ik", rotations, rotations) - np.eye(3)).max(axis=(-2, -1)) + + +@check("valid_rotations") +def valid_rotations(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode with an observed pose rotation that is not orthonormal or a root quaternion that is not unit. + + Both tests hold for either quaternion order, so the check needs no convention. + """ + bad = [] + for path, dataset in episode.datasets(): + if path.startswith(OBSERVED_POSE_PATHS) and dataset.shape[-2:] == (4, 4): + error = _rotation_errors(dataset[()]).max(initial=0.0) + elif path.endswith("root_pose") and dataset.shape[-1] == 7: + error = np.abs(np.linalg.norm(dataset[()][..., 3:7], axis=-1) - 1.0).max(initial=0.0) + else: + continue + if error > ROTATION_TOLERANCE: + bad.append(path) + return f"invalid rotations in {', '.join(bad)}" if bad else None + + +@check("pose_action_bounds", requires=["env_args.autodata.pose_action"]) +def pose_action_bounds(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode whose pose actions exceed the bound the embodiment clips them to.""" + actions = episode["actions"] + bad = [] + for eef, layout in ev.entry["pose_action"].items(): + if layout["bound"] is None: + continue + start, end = layout["action_slice"] + peak = float(np.abs(actions[:, start:end]).max(initial=0.0)) + if peak > layout["bound"] + 1e-6: + bad.append(f"{eef} reaches {peak:.3g}") + return f"pose actions beyond their bound: {', '.join(bad)}" if bad else None + + +@check("one_frame_per_step", group="image") +def one_frame_per_step(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode whose camera streams do not hold one frame per action.""" + num_steps = len(episode["actions"]) + bad = [path for path, dataset in _image_streams(episode) if len(dataset) != num_steps] + return f"{', '.join(bad)} do not hold {num_steps} frames" if bad else None + + +@check("no_black_frames", group="image") +def no_black_frames(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode with a camera frame whose mean pixel value is below :data:`BLACK_FRAME_MEAN`.""" + for path, dataset in _image_streams(episode): + for start in range(0, len(dataset), FRAME_CHUNK): + means = dataset[start : start + FRAME_CHUNK].mean(axis=(1, 2, 3)) + if (means < BLACK_FRAME_MEAN).any(): + return f"{path} is black at step {start + int(np.argmax(means < BLACK_FRAME_MEAN))}" + return None + + +@check("no_frozen_frames", group="image") +def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: + """Fails an episode whose camera repeats one frame :data:`FROZEN_FRAMES` times in a row.""" + for path, dataset in _image_streams(episode): + previous, repeats = None, 1 + for start in range(0, len(dataset), FRAME_CHUNK): + for offset, frame in enumerate(dataset[start : start + FRAME_CHUNK]): + repeats = repeats + 1 if previous is not None and np.array_equal(frame, previous) else 1 + if repeats >= FROZEN_FRAMES: + return f"{path} repeats one frame {FROZEN_FRAMES} times up to step {start + offset}" + previous = frame + return None + + +def _pass_rate(ev: Evidence, group: str) -> Value: + names = [name for name, check_group in ev.check_groups.items() if check_group == group] + if not names: + raise NotApplicable(f"no {group} check applies to this dataset") + if not ev.all_episodes: + raise NotApplicable("the dataset has no episodes") + failing = {episode for name in names for episode in ev.failures[name]} + details = {f"{name}_failed": len(ev.failures[name]) for name in names} + return Value(1 - len(failing) / len(ev.all_episodes), n=len(ev.all_episodes), details=details) + + +@metric( + "correctness_pass_rate", + area="validity", + signal="data correctness", + scope="episode", + unit="fraction", + better="higher", +) +def correctness_pass_rate(ev: Evidence) -> Value: + """Share of episodes passing every correctness check: finite, aligned, valid rotations, actions in bounds.""" + return _pass_rate(ev, "correctness") + + +@metric( + "image_stream_pass_rate", + area="validity", + signal="data correctness", + scope="episode", + unit="fraction", + better="higher", +) +def image_stream_pass_rate(ev: Evidence) -> Value: + """Share of episodes whose camera streams hold one frame per step, no black frame and no frozen run.""" + return _pass_rate(ev, "image") + + +@metric( + "final_success_rate", + area="validity", + signal="task correctness", + scope="dataset", + unit="fraction", + better="higher", + requires=["step_success"], + baseline=True, +) +def final_success_rate(ev: Evidence) -> Value: + """Share of valid episodes whose task success holds at their last step.""" + held = [bool(episode["step_success"][-1]) for episode in ev.episodes] + return Value(sum(held) / len(held), n=len(held)) + + +@metric( + "action_rotation_error", + area="validity", + signal="data correctness", + scope="dataset", + unit="max |R^T R - I|", + better="lower", + primary="p95", + requires=["obs/datagen_info/target_eef_pose"], + baseline=True, +) +def action_rotation_error(ev: Evidence) -> Value: + """p50 and p95 deviation of the commanded target rotations from a valid rotation, over every step and end effector.""" + errors = np.concatenate([ + _rotation_errors(episode[f"obs/datagen_info/target_eef_pose/{eef}"]) + for episode in ev.episodes + for eef in episode.children("obs/datagen_info/target_eef_pose") + ]) + if not len(errors): + raise NotApplicable("no end effector commands a pose") + p50, p95 = np.percentile(errors, [50, 95]) + return Value({"p50": float(p50), "p95": float(p95)}, n=len(errors)) diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index 08d996f..039fc56 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -12,8 +12,15 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 -# Metrics a MimicGen run reported with its result file must compute. -RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset"} +# Metrics a MimicGen run recorded with --record_metric_inputs must compute; no camera, so no image checks. +METRICS = { + "generation_success_rate", + "demos_per_gpu_hour", + "time_to_dataset", + "correctness_pass_rate", + "final_success_rate", + "action_rotation_error", +} def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: @@ -65,7 +72,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: "--report_file", report_file, ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(report_file, METRICS) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index b464095..ed5f10a 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -21,13 +21,16 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 -# Metrics a SkillGen run reported with its result file must compute; the planning metrics are SkillGen's own. -RUN_METRICS = { +# Metrics a SkillGen run recorded with --record_metric_inputs must compute; the planning metrics are SkillGen's own. +METRICS = { "generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "planning_time", "planning_failure_rate", + "correctness_pass_rate", + "final_success_rate", + "action_rotation_error", } @@ -85,7 +88,7 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: "--report_file", report_file, ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(report_file, METRICS) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py new file mode 100644 index 0000000..a252907 --- /dev/null +++ b/autodata_tests/metrics/test_validity.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known-answer tests of the episode checks and validity metrics, each on episodes with one known defect.""" + +import h5py +import json +import numpy as np +from pathlib import Path + +import pytest + +from autodata_metrics import REGISTERED_METRICS, build_report +from autodata_metrics.validity import FROZEN_FRAMES + + +def _write(path: Path, episodes: list[dict[str, np.ndarray]], entry: dict | None = None) -> str: + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + env_args = {"env_name": "Test-v0", **({"autodata": entry} if entry else {})} + data.attrs["env_args"] = json.dumps(env_args) + for index, arrays in enumerate(episodes): + group = data.create_group(f"demo_{index}") + for key, array in arrays.items(): + group.create_dataset(key, data=array) + return str(path) + + +def _metrics(*names: str) -> dict: + return {name: REGISTERED_METRICS[name] for name in names} + + +def test_failing_episodes_are_counted_and_left_out_of_dataset_metrics(tmp_path): + actions = np.zeros((3, 7)) + good = {"actions": actions, "step_success": np.array([False, False, True])} + non_finite = {"actions": np.where(np.eye(3, 7) > 0, np.nan, 0.0), "step_success": np.array([True, True, False])} + short = {"actions": actions, "step_success": np.array([False, False])} + + report = build_report( + _write(tmp_path / "d.hdf5", [good, non_finite, short]), + metrics=_metrics("correctness_pass_rate", "final_success_rate"), + ) + + assert report["metrics"]["correctness_pass_rate"]["value"] == pytest.approx(1 / 3) + assert report["metrics"]["final_success_rate"]["value"] == 1.0 and report["metrics"]["final_success_rate"]["n"] == 1 + assert {name: entry["failed"] for name, entry in report["check_failures"].items() if entry["failed"]} == { + "finite": 1, + "aligned_lengths": 1, + } + + +def test_rotation_and_bound_checks_flag_their_defects(tmp_path): + pose = np.tile(np.eye(4), (3, 1, 1)) + entry = {"run_id": "r", "exported_episodes": 3, "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0}}} + clipped = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose} + beyond_bound = {"actions": np.full((3, 7), 1.5), "obs/datagen_info/eef_pose/arm": pose} + skewed = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose * [[1.1], [1.0], [1.0], [1.0]]} + + report = build_report(_write(tmp_path / "d.hdf5", [clipped, beyond_bound, skewed], entry), metrics={}) + + failures = {name: sorted(entry["examples"]) for name, entry in report["check_failures"].items() if entry["failed"]} + assert failures == {"pose_action_bounds": ["demo_1"], "valid_rotations": ["demo_2"]} + + +def test_image_checks_flag_black_frozen_and_missing_frames(tmp_path): + steps = FROZEN_FRAMES + 2 + rng = np.random.default_rng(0) + + def frames(count: int) -> np.ndarray: + return rng.integers(50, 200, size=(count, 4, 4, 3), dtype=np.uint8) + + black = frames(steps) + black[1] = 0 + frozen = np.repeat(frames(1), steps, axis=0) + episodes = [ + {"actions": np.zeros((steps, 7)), "obs/camera": camera} + for camera in (frames(steps), black, frozen, frames(steps - 1)) + ] + + report = build_report(_write(tmp_path / "d.hdf5", episodes), metrics=_metrics("image_stream_pass_rate")) + + failures = {name: sorted(entry["examples"]) for name, entry in report["check_failures"].items() if entry["failed"]} + assert failures == {"no_black_frames": ["demo_1"], "no_frozen_frames": ["demo_2"], "one_frame_per_step": ["demo_3"]} + assert report["metrics"]["image_stream_pass_rate"]["value"] == pytest.approx(1 / 4) + + +def test_final_success_rate_reports_the_source_baseline(tmp_path): + ending = [{"actions": np.zeros((2, 7)), "step_success": np.array([True, held])} for held in (True, False)] + source = _write(tmp_path / "source.hdf5", ending[:1]) + + row = build_report(_write(tmp_path / "d.hdf5", ending), source=source, metrics=_metrics("final_success_rate"))[ + "metrics" + ]["final_success_rate"] + + assert (row["value"], row["baseline"]) == (0.5, 1.0) + + +def test_action_rotation_error_measures_commanded_rotations(tmp_path): + identity = np.tile(np.eye(4), (4, 1, 1)) + episode = { + "actions": np.zeros((4, 7)), + "obs/datagen_info/eef_pose/arm": identity, + "obs/datagen_info/target_eef_pose/arm": identity * [[1.01], [1.0], [1.0], [1.0]], + # A channel that commands no pose records an all-zero target; it is not a rotation and is not counted. + "obs/datagen_info/target_eef_pose/body": np.zeros((4, 4, 4)), + } + + report = build_report(_write(tmp_path / "d.hdf5", [episode]), metrics=_metrics("action_rotation_error")) + + row = report["metrics"]["action_rotation_error"] + assert row["value"]["p95"] == pytest.approx(1.01**2 - 1) and row["n"] == 4 + assert report["num_valid_episodes"] == 1 diff --git a/docs/_ext/autodata_doc_tools.py b/docs/_ext/autodata_doc_tools.py index 72db1fd..1ef9432 100644 --- a/docs/_ext/autodata_doc_tools.py +++ b/docs/_ext/autodata_doc_tools.py @@ -71,6 +71,11 @@ def curobo_replacer(_: Any) -> str: source[0] = re.sub(r":docker_run_curobo:", curobo_replacer, source[0]) +def _first_line(docstring: str | None) -> str: + """The docstring's summary line, with Sphinx roles turned into literals that need no reference target.""" + return re.sub(r":\w+:`~?([^`]+)`", r"``\1``", (docstring or "").split("\n")[0]) + + def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: """Replaces ``:autodata_metrics_table:`` with a table of the built-in metrics. @@ -87,8 +92,8 @@ def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: for decorator in node.decorator_list: if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "metric": options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} - requires = ", ".join(f"``{key}``" for key in options.get("requires", ())) or "-" - definition = (ast.get_docstring(node) or "").split("\n")[0] + requires = ", ".join(f"``{key}``" for key in options.get("requires", ())) or "none" + definition = _first_line(ast.get_docstring(node)) name = ast.literal_eval(decorator.args[0]) rows.append((options["area"], name, options["unit"], requires, definition)) table = [ @@ -99,13 +104,41 @@ def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: " * - Metric\n - Area\n - Unit\n - Requires\n - Definition", ] for area, name, unit, requires, definition in sorted(rows): + unit, definition = (text.replace("|", "\\|") for text in (unit, definition)) table.append(f" * - ``{name}``\n - {area}\n - {unit}\n - {requires}\n - {definition}") source[0] = source[0].replace(":autodata_metrics_table:", "\n".join(table) + "\n") +def autodata_checks_table(app: Sphinx, _: Any, source: list[str]) -> None: + """Replaces ``:autodata_checks_table:`` with a table of the episode checks, read like the metrics table.""" + if ":autodata_checks_table:" not in source[0]: + return + rows = [] + for path in sorted((Path(app.confdir).parent / "autodata_metrics").glob("*.py")): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if not isinstance(node, ast.FunctionDef): + continue + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "check": + options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} + definition = _first_line(ast.get_docstring(node)) + rows.append((options.get("group", "correctness"), ast.literal_eval(decorator.args[0]), definition)) + table = [ + ".. list-table::", + " :header-rows: 1", + " :widths: 22 14 64", + "", + " * - Check\n - Group\n - Definition", + ] + for group, name, definition in sorted(rows): + table.append(f" * - ``{name}``\n - {group}\n - {definition}") + source[0] = source[0].replace(":autodata_checks_table:", "\n".join(table) + "\n") + + def setup(app: Sphinx) -> None: app.connect("source-read", autodata_git_clone_code_block) app.connect("source-read", autodata_code_link) app.connect("source-read", docker_run_command_replacer) app.connect("source-read", autodata_metrics_table) + app.connect("source-read", autodata_checks_table) app.add_config_value("autodata_docs_config", {}, "env") diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index b327d61..dfa7591 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -35,6 +35,15 @@ Gating A gated metric also violates the gate if it is not applicable, fails, or reports a different ``version``. +Episode checks +-------------- + +Before any metric, every episode goes through the checks below. An episode that fails one is counted in +``correctness_pass_rate`` or ``image_stream_pass_rate`` and left out of every dataset metric; the report lists +the failures per check, with examples. + +:autodata_checks_table: + Built-in metrics ---------------- From 2bd83526b9cf88883c58cb029a44f908c7a8c791 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 20:29:40 -0500 Subject: [PATCH 12/24] fix: check the attempt log against its result and keep reports valid JSON - The report refuses a result whose attempt and success counts differ from its attempt log. - Run and gate input checks raise ValueError, so they also hold under python -O. - Non-finite numbers in a metric's details are reported as null. - A dataset whose env_args are not a JSON object reads as having none. - An empty metrics mapping computes no metrics instead of all of them. - Docs: how the report treats a result from another run; the example metric gets its own name; joint-limit metrics are not mentioned yet. - The end-to-end tests run the report through assert_ok_metrics. --- autodata_metrics/evidence.py | 26 ++++------ autodata_metrics/report.py | 52 ++++++++++++++----- .../e2e/test_mimicgen_data_generation.py | 13 +---- .../e2e/test_skillgen_bin_data_generation.py | 13 +---- autodata_tests/metrics/test_report.py | 18 +++++-- autodata_tests/utils/utils.py | 22 +++++--- docs/pages/references/metrics.rst | 25 ++++----- scripts/validate_dataset.py | 9 +--- 8 files changed, 94 insertions(+), 84 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 972b628..5f5a8b2 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -11,27 +11,21 @@ from collections.abc import Mapping from typing import Any -_MISSING = object() - -def lookup(record: Any, dotted_key: str) -> Any: - """``record[a][b]`` for ``dotted_key`` ``"a.b"``, or a sentinel that :func:`is_present` rejects.""" +def _has_path(record: Any, dotted_key: str) -> bool: + """Whether ``record[a][b]`` exists for ``dotted_key`` ``"a.b"``.""" for part in dotted_key.split("."): if not isinstance(record, Mapping) or part not in record: - return _MISSING + return False record = record[part] - return record + return True def read_env_args(h5_file: h5py.File) -> dict[str, Any]: - """The dataset's ``env_args``; its ``autodata`` entry was written by the producer.""" + """The dataset's ``env_args``, or empty when they are not a JSON object.""" raw_env_args = h5_file["data"].attrs.get("env_args", "{}") - return json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) - - -def is_present(value: Any) -> bool: - """Whether a :func:`lookup` found its key.""" - return value is not _MISSING + env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + return env_args if isinstance(env_args, dict) else {} class Episode: @@ -88,9 +82,9 @@ def has(self, key: str) -> bool: if head == "source": return self.source is not None and (not rest or self.source.has(rest)) if head == "attempts": - return bool(self.attempts) and (not rest or all(is_present(lookup(line, rest)) for line in self.attempts)) + return bool(self.attempts) and (not rest or all(_has_path(line, rest) for line in self.attempts)) if head == "env_args": - return is_present(lookup(self.env_args, rest)) + return _has_path(self.env_args, rest) if head == "run": - return is_present(lookup(self.run, rest)) + return _has_path(self.run, rest) return bool(self.episodes) and all(key in episode for episode in self.episodes) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 41d7133..0a80c06 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -54,23 +54,36 @@ def check_same_run(evidence: Evidence) -> list[str]: Run ids alone are not enough: merging datasets keeps the first file's ``env_args``, so the exported episode count must match too. + + Raises: + ValueError: If the dataset, result and attempt log disagree on their run or their counts. """ num_episodes = len(evidence.episodes) - if evidence.result is not None: - assert ( - num_episodes == evidence.result["num_success"] - ), f"The dataset holds {num_episodes} episodes but the run counts {evidence.result['num_success']} successes." + result = evidence.result + if result is not None and num_episodes != result["num_success"]: + raise ValueError( + f"The dataset holds {num_episodes} episodes but the run counts {result['num_success']} successes." + ) + if result is not None and evidence.attempts is not None: + logged = len(evidence.attempts), sum(line["success"] for line in evidence.attempts) + if logged != (result["num_attempts"], result["num_success"]): + raise ValueError( + f"The attempt log holds {logged[0]} attempts and {logged[1]} successes but the result counts" + f" {result['num_attempts']} and {result['num_success']}." + ) entry = evidence.env_args.get("autodata") if entry is None: return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] - assert entry["exported_episodes"] == num_episodes, ( - f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" - " it was merged or edited after export." - ) + if entry["exported_episodes"] != num_episodes: + raise ValueError( + f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" + " it was merged or edited after export." + ) run_ids = {entry["run_id"]} | {line["run_id"] for line in evidence.attempts or []} if "run_id" in evidence.run: run_ids.add(evidence.run["run_id"]) - assert len(run_ids) == 1, f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}" + if len(run_ids) != 1: + raise ValueError(f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}") return [] @@ -96,7 +109,7 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} if not _is_finite(value.value): return {**row, "status": "error", "reason": f"non-finite value {value.value}"} - return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + return {**row, "status": "ok", "value": value.value, "n": value.n, "details": _finite_or_none(value.details)} def _is_finite(value: Any) -> bool: @@ -106,6 +119,16 @@ def _is_finite(value: Any) -> bool: return isinstance(value, numbers.Real) and math.isfinite(value) +def _finite_or_none(value: Any) -> Any: + """``value`` with every non-finite number replaced by None, so the report stays valid JSON.""" + value = value.tolist() if hasattr(value, "tolist") else value + if isinstance(value, Mapping): + return {key: _finite_or_none(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_finite_or_none(item) for item in value] + return None if isinstance(value, float) and not math.isfinite(value) else value + + def build_report( dataset: str, result_file: str | None = None, @@ -127,7 +150,8 @@ def build_report( source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) - rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} + metrics = REGISTERED_METRICS if metrics is None else metrics + rows = {name: evaluate(spec, evidence) for name, spec in metrics.items()} num_episodes = len(evidence.episodes) return { "schema_version": SCHEMA_VERSION, @@ -150,6 +174,9 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or at another version is a violation. + + Raises: + ValueError: If a gate entry bounds a dict value without a ``key`` and the metric has no primary entry. """ violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): @@ -162,7 +189,8 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: continue key = bound.get("key", row["primary"]) value = row["value"].get(key) if key else row["value"] - assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if isinstance(value, Mapping): + raise ValueError(f"Gate entry {name!r} needs a key into {sorted(value)}.") if not _is_finite(value): violations.append(f"{name}: {value!r} is not a finite number") continue diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index 08d996f..16b27f1 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -54,18 +54,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) assert_recorded_run(output_file, result_file) - report_file = os.path.join(temp_dir, "report.json") - run_subprocess([ - TestPaths.python_path, - os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), - output_file, - "--report", - "--result_file", - result_file, - "--report_file", - report_file, - ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(output_file, result_file, RUN_METRICS) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index b464095..bf45200 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -74,18 +74,7 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: attempts = assert_recorded_run(output_file, result_file) # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) - report_file = os.path.join(temp_dir, "report.json") - run_subprocess([ - TestPaths.python_path, - os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), - output_file, - "--report", - "--result_file", - result_file, - "--report_file", - report_file, - ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(output_file, result_file, RUN_METRICS) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 69579ac..1071cb9 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -82,11 +82,16 @@ def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): assert "belongs to another run" in report["warnings"][0] -def test_report_refuses_merged_data(tmp_path): +def test_report_refuses_records_whose_counts_disagree(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} - with pytest.raises(AssertionError, match="merged or edited"): + with pytest.raises(ValueError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + result_file = _run(tmp_path, "run-a") + _log(tmp_path, "run-a", [True, True]) + with pytest.raises(ValueError, match="attempt log holds 2 attempts"): + build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=result_file, metrics={}) + def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} @@ -102,13 +107,18 @@ def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] -def test_non_finite_value_is_an_error_and_fails_the_gate(tmp_path): - report = build_report(_dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(float("nan"), n=1))}) +def test_non_finite_numbers_never_reach_the_report(tmp_path): + metrics = { + "m": _spec("m", lambda ev: Value(float("nan"), n=1)), + "d": _spec("d", lambda ev: Value(1.0, n=1, details={"interval_95": [0.5, float("inf")]})), + } + report = build_report(_dataset(tmp_path / "d.hdf5"), metrics=metrics) gate = tmp_path / "gate.yaml" gate.write_text("m: {min: 0}\n") assert report["metrics"]["m"]["status"] == "error" assert gate_violations(report, str(gate)) == ["m: error"] + assert report["metrics"]["d"]["details"] == {"interval_95": [0.5, None]} def test_metrics_import_without_torch_or_isaaclab(): diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index 8e15b97..14a47b6 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -7,6 +7,9 @@ import json import os +from autodata_tests.utils.constants import TestPaths +from autodata_tests.utils.subprocess import run_subprocess + def assert_valid_dataset(output_file: str, min_num_demos: int) -> None: """Assert the run wrote a dataset containing at least min_num_demos entries.""" @@ -87,21 +90,28 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: return attempts -def assert_ok_metrics(report_file: str, expected_ok: set[str]) -> dict: - """Assert exactly ``expected_ok`` metrics computed a value; any other metric must say why it did not. +def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str]) -> None: + """Report a generation run's metrics with ``validate_dataset.py`` and assert exactly ``expected_ok`` computed a value. A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which this catches. - - Returns: - The report's metric rows. """ + report_file = os.path.join(os.path.dirname(dataset_file), "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + dataset_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) with open(report_file, encoding="utf-8") as report_handle: rows = json.load(report_handle)["metrics"] ok = {name for name, row in rows.items() if row["status"] == "ok"} assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" - return rows def assert_no_metric_inputs(dataset_file: str) -> None: diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index b327d61..18a9ff8 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -19,8 +19,10 @@ Each metric reports one of three things: - ``not_applicable``, with the reason, such as a missing recorded input; - ``error``, when it fails. The other metrics are still computed. -The report refuses a dataset and a result file from different runs, because their run ids and exported -episode counts must match. +The report reads the run named by the dataset's run id. A result file from another run, such as the +earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own +attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success +counts disagree, as they do after datasets are merged. Gating ------ @@ -48,26 +50,22 @@ requires. Without that evidence, the report says which key is missing and does n .. code-block:: python - import numpy as np - from autodata_metrics import Value, metric @metric( - "planning_time", + "slowest_plan", area="diagnostics", signal="generation performance", scope="run", - unit="s per attempt", + unit="s", better="lower", - primary="p95", requires=["attempts.extras.planning_time_s"], ) - def planning_time(ev): - """p50 and p95 motion-planning time per attempt.""" - times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) - p50, p95 = np.percentile(times, [50, 95]) - return Value({"p50": float(p50), "p95": float(p95)}, n=len(times)) + def slowest_plan(ev): + """Longest motion-planning time of any attempt.""" + times = [attempt["extras"]["planning_time_s"] for attempt in ev.attempts] + return Value(max(times), n=len(times)) An evidence key is one of: @@ -83,7 +81,6 @@ When you add an algorithm or an embodiment: - An algorithm reports per-attempt facts by overriding ``attempt_extras()``. Metrics that require those facts then apply to its runs. -- An embodiment's adapter describes its pose actions through ``metric_context()``. Add datasheet - ``hardware_limits`` to its YAML to enable the joint-limit metrics. +- An embodiment's adapter describes its pose actions through ``metric_context()``. - List the workflow's expected metrics in its end-to-end test, so a metric that stops applying fails the test. diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index fd8aa08..8b9bb43 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -178,14 +178,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.report and len(args.files) != 1: parser.error("--report takes exactly one dataset") - report_options = { - "--result_file": args.result_file, - "--source": args.source, - "--metrics": args.metrics, - "--gate": args.gate, - "--report_file": args.report_file, - } - given = [option for option, value in report_options.items() if value] + given = [f"--{name}" for name in ("result_file", "source", "metrics", "gate", "report_file") if getattr(args, name)] if given and not args.report: parser.error(f"{', '.join(given)} only apply with --report") From 45d5ca4fd8de82522a98b37b9d78f398f56c34b8 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 20:32:31 -0500 Subject: [PATCH 13/24] refactor: compute baselines on the source as is and share the docs table code - No baseline metric reads the run's autodata entry or control step, so the source dataset is evaluated as it is. - One parser of the @metric and @check declarations builds both docs tables. - Episode.datasets returns a list; the module constants sit together. --- autodata_metrics/evidence.py | 30 ++--------- autodata_metrics/registry.py | 2 +- autodata_metrics/report.py | 8 +-- autodata_metrics/validity.py | 14 ++--- docs/_ext/autodata_doc_tools.py | 92 +++++++++++++++------------------ 5 files changed, 56 insertions(+), 90 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index cb11596..e6c3bcb 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -5,11 +5,10 @@ from __future__ import annotations -import copy import h5py import json import numpy as np -from collections.abc import Iterator, Mapping +from collections.abc import Mapping from typing import Any @@ -46,13 +45,13 @@ def children(self, key: str) -> list[str]: """Names under the group ``key``, such as the end effectors under ``obs/datagen_info/eef_pose``.""" return list(self._group[key]) - def datasets(self) -> Iterator[tuple[str, h5py.Dataset]]: + def datasets(self) -> list[tuple[str, h5py.Dataset]]: """Every dataset of the episode with its path, unread until accessed.""" found: list[tuple[str, h5py.Dataset]] = [] self._group.visititems( lambda path, item: found.append((path, item)) if isinstance(item, h5py.Dataset) else None ) - yield from found + return found class Evidence: @@ -107,26 +106,3 @@ def has(self, key: str) -> bool: if head == "run": return _has_path(self.run, rest) return bool(self.episodes) and all(key in episode for episode in self.episodes) - - @property - def entry(self) -> Mapping[str, Any]: - """The ``autodata`` entry of ``env_args``, or empty.""" - return self.env_args.get("autodata", {}) - - @property - def step_dt(self) -> float | None: - """Control step [s]: the simulation step times the decimation, or None when not recorded.""" - sim_args = self.env_args.get("sim_args") or {} - return float(sim_args["dt"]) * int(sim_args["decimation"]) if {"dt", "decimation"} <= set(sim_args) else None - - def as_baseline_for(self, run: Evidence) -> Evidence: - """This source dataset as the baseline of ``run``: its own episodes, with the run's entry and control step. - - Shipped sources predate the ``autodata`` entry, so a metric reads the robot's layout and limits from the run. - """ - baseline = copy.copy(self) - baseline.env_args = {**self.env_args, "autodata": run.entry} - if self.step_dt is None and "sim_args" in run.env_args: - baseline.env_args["sim_args"] = run.env_args["sim_args"] - baseline.result, baseline.run, baseline.attempts, baseline.source = None, {}, None, None - return baseline diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py index b812429..433c687 100644 --- a/autodata_metrics/registry.py +++ b/autodata_metrics/registry.py @@ -97,7 +97,7 @@ def metric( requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`. primary: For a dict value, the entry a gate checks by default. version: Raise it whenever the definition changes, so gates pinned to the old one fail. - baseline: Also compute the metric on the source dataset, with the generated run's context. + baseline: Also compute the metric on the source dataset. """ assert area in AREAS, f"Metric {name!r} has area {area!r}; expected one of {AREAS}." diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 2a3a6a7..db5dfa5 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -58,7 +58,7 @@ def check_same_run(evidence: Evidence) -> list[str]: Raises: ValueError: If the dataset, result and attempt log disagree on their run or their counts. """ - num_episodes = len(evidence.episodes) + num_episodes = len(evidence.all_episodes) result = evidence.result if result is not None and num_episodes != result["num_success"]: raise ValueError( @@ -111,7 +111,7 @@ def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: evidence.episodes = [episode for episode in evidence.all_episodes if episode.name not in failing] -def evaluate(spec: MetricSpec, evidence: Evidence, with_baseline: bool = True) -> dict[str, Any]: +def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: """One report row: the metric's value, or why it is not applicable or failed.""" row = { "area": spec.area, @@ -136,8 +136,8 @@ def evaluate(spec: MetricSpec, evidence: Evidence, with_baseline: bool = True) - if not _is_finite(value.value): return {**row, "status": "error", "reason": f"non-finite value {value.value}"} row = {**row, "status": "ok", "value": value.value, "n": value.n, "details": _finite_or_none(value.details)} - if with_baseline and spec.baseline and evidence.source is not None: - row["baseline"] = evaluate(spec, evidence.source.as_baseline_for(evidence), with_baseline=False).get("value") + if spec.baseline and evidence.source is not None: + row["baseline"] = evaluate(spec, evidence.source).get("value") return row diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 1344550..0751aef 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -19,6 +19,10 @@ """Identical consecutive camera frames that count as a frozen stream.""" FRAME_CHUNK = 64 """Camera frames read at a time, so image checks stream instead of loading whole episodes.""" +PER_EPISODE_PATHS = ("initial_state", "obs/datagen_info/subtask_term_signals", "obs/datagen_info/subtask_start_signals") +"""Paths whose arrays are not one entry per step: the initial state, and annotation's subtask signals.""" +OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") +"""Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" def _is_image(dataset: h5py.Dataset) -> bool: @@ -41,10 +45,6 @@ def finite(episode: Episode, ev: Evidence) -> str | None: return f"non-finite values in {', '.join(bad)}" if bad else None -PER_EPISODE_PATHS = ("initial_state", "obs/datagen_info/subtask_term_signals", "obs/datagen_info/subtask_start_signals") -"""Paths whose arrays are not one entry per step: the initial state, and annotation's subtask signals.""" - - @check("aligned_lengths") def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: """Fails an episode whose per-step arrays do not all hold one entry per action; camera streams are checked apart.""" @@ -59,10 +59,6 @@ def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: return f"{', '.join(bad)} do not hold {num_steps} steps" if bad else None -OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") -"""Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" - - def _rotation_errors(poses: np.ndarray) -> np.ndarray: """Per pose, the largest entry of ``|R^T R - I|``; all-zero poses, a channel's placeholder target, are skipped.""" rotations = poses[np.abs(poses).reshape(len(poses), -1).max(axis=1) > 0][..., :3, :3] @@ -93,7 +89,7 @@ def pose_action_bounds(episode: Episode, ev: Evidence) -> str | None: """Fails an episode whose pose actions exceed the bound the embodiment clips them to.""" actions = episode["actions"] bad = [] - for eef, layout in ev.entry["pose_action"].items(): + for eef, layout in ev.env_args["autodata"]["pose_action"].items(): if layout["bound"] is None: continue start, end = layout["action_slice"] diff --git a/docs/_ext/autodata_doc_tools.py b/docs/_ext/autodata_doc_tools.py index 1ef9432..f44d232 100644 --- a/docs/_ext/autodata_doc_tools.py +++ b/docs/_ext/autodata_doc_tools.py @@ -71,68 +71,62 @@ def curobo_replacer(_: Any) -> str: source[0] = re.sub(r":docker_run_curobo:", curobo_replacer, source[0]) -def _first_line(docstring: str | None) -> str: - """The docstring's summary line, with Sphinx roles turned into literals that need no reference target.""" - return re.sub(r":\w+:`~?([^`]+)`", r"``\1``", (docstring or "").split("\n")[0]) +def _declarations(app: Sphinx, decorator: str) -> list[tuple[str, dict[str, Any], str]]: + """Name, options and summary line of every function in ``autodata_metrics/`` registered with ``decorator``. - -def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: - """Replaces ``:autodata_metrics_table:`` with a table of the built-in metrics. - - The rows are read from the ``@metric`` declarations in ``autodata_metrics/`` instead of importing them, so the - docs build needs none of the metrics' dependencies and the table cannot drift from the code. + The declarations are parsed instead of imported, so the docs build needs none of the metrics' dependencies and + the tables cannot drift from the code. In the summary, Sphinx roles become literals that need no reference + target, and pipes are escaped so they do not read as substitutions. """ - if ":autodata_metrics_table:" not in source[0]: - return - rows = [] + found = [] for path in sorted((Path(app.confdir).parent / "autodata_metrics").glob("*.py")): for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): if not isinstance(node, ast.FunctionDef): continue - for decorator in node.decorator_list: - if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "metric": - options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} - requires = ", ".join(f"``{key}``" for key in options.get("requires", ())) or "none" - definition = _first_line(ast.get_docstring(node)) - name = ast.literal_eval(decorator.args[0]) - rows.append((options["area"], name, options["unit"], requires, definition)) - table = [ - ".. list-table::", - " :header-rows: 1", - " :widths: 20 12 14 26 28", - "", - " * - Metric\n - Area\n - Unit\n - Requires\n - Definition", + for call in node.decorator_list: + if isinstance(call, ast.Call) and getattr(call.func, "id", None) == decorator: + options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in call.keywords} + summary = (ast.get_docstring(node) or "").split("\n")[0] + summary = re.sub(r":\w+:`~?([^`]+)`", r"``\1``", summary).replace("|", "\\|") + found.append((ast.literal_eval(call.args[0]), options, summary)) + return found + + +def _list_table(widths: str, header: tuple[str, ...], rows: list[tuple[str, ...]]) -> str: + lines = [".. list-table::", " :header-rows: 1", f" :widths: {widths}", ""] + lines += [" * - " + "\n - ".join(row) for row in [header, *rows]] + return "\n".join(lines) + "\n" + + +def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: + """Replaces ``:autodata_metrics_table:`` with a table of the built-in metrics.""" + if ":autodata_metrics_table:" not in source[0]: + return + rows = [ + ( + f"``{name}``", + options["area"], + options["unit"].replace("|", "\\|"), + ", ".join(f"``{key}``" for key in options.get("requires", ())) or "none", + summary, + ) + for name, options, summary in sorted(_declarations(app, "metric"), key=lambda item: (item[1]["area"], item[0])) ] - for area, name, unit, requires, definition in sorted(rows): - unit, definition = (text.replace("|", "\\|") for text in (unit, definition)) - table.append(f" * - ``{name}``\n - {area}\n - {unit}\n - {requires}\n - {definition}") - source[0] = source[0].replace(":autodata_metrics_table:", "\n".join(table) + "\n") + table = _list_table("20 12 14 26 28", ("Metric", "Area", "Unit", "Requires", "Definition"), rows) + source[0] = source[0].replace(":autodata_metrics_table:", table) def autodata_checks_table(app: Sphinx, _: Any, source: list[str]) -> None: - """Replaces ``:autodata_checks_table:`` with a table of the episode checks, read like the metrics table.""" + """Replaces ``:autodata_checks_table:`` with a table of the episode checks.""" if ":autodata_checks_table:" not in source[0]: return - rows = [] - for path in sorted((Path(app.confdir).parent / "autodata_metrics").glob("*.py")): - for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): - if not isinstance(node, ast.FunctionDef): - continue - for decorator in node.decorator_list: - if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "check": - options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} - definition = _first_line(ast.get_docstring(node)) - rows.append((options.get("group", "correctness"), ast.literal_eval(decorator.args[0]), definition)) - table = [ - ".. list-table::", - " :header-rows: 1", - " :widths: 22 14 64", - "", - " * - Check\n - Group\n - Definition", + declarations = [ + (options.get("group", "correctness"), name, summary) for name, options, summary in _declarations(app, "check") ] - for group, name, definition in sorted(rows): - table.append(f" * - ``{name}``\n - {group}\n - {definition}") - source[0] = source[0].replace(":autodata_checks_table:", "\n".join(table) + "\n") + rows = [(f"``{name}``", group, summary) for group, name, summary in sorted(declarations)] + source[0] = source[0].replace( + ":autodata_checks_table:", _list_table("22 14 64", ("Check", "Group", "Definition"), rows) + ) def setup(app: Sphinx) -> None: From 475e598247aaf88d53f4ea9db9970967c7346cef Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 20:44:44 -0500 Subject: [PATCH 14/24] fix: refuse a source other than the run's and non-finite gate bounds - When the result records the run's source, the report refuses a --source with a different SHA-256, so source-based metrics read the dataset the run started from. - A gate entry whose min or max is not a finite number is an error instead of a bound that can never be violated. --- autodata_metrics/report.py | 15 +++++++++++++-- autodata_tests/metrics/test_report.py | 18 ++++++++++++++---- docs/pages/references/metrics.rst | 5 +++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 0a80c06..d77d285 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -142,6 +142,9 @@ def build_report( result_file: Generation result (``generate_dataset.py --result_file``); its attempt log is read beside it. source: Source dataset the run started from. metrics: Metrics to compute; defaults to every registered metric. + + Raises: + ValueError: If the dataset, result, attempt log and source do not all come from one run. """ with contextlib.ExitStack() as stack: h5_file = stack.enter_context(h5py.File(dataset, "r")) @@ -150,6 +153,10 @@ def build_report( source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) + source_record = input_record(source) + recorded_source = evidence.run.get("inputs", {}).get("source") + if source_record and recorded_source and source_record["sha256"] != recorded_source["sha256"]: + raise ValueError(f"The source {source} is not the one the run started from, {recorded_source['path']}.") metrics = REGISTERED_METRICS if metrics is None else metrics rows = {name: evaluate(spec, evidence) for name, spec in metrics.items()} num_episodes = len(evidence.episodes) @@ -158,7 +165,7 @@ def build_report( "inputs": { "dataset": input_record(dataset), "result_file": result_file, - "source": input_record(source), + "source": source_record, "run_id": run_id, "revision": code_revision(), }, @@ -176,10 +183,14 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: at another version is a violation. Raises: - ValueError: If a gate entry bounds a dict value without a ``key`` and the metric has no primary entry. + ValueError: If a gate entry has a bound that is not a finite number, or bounds a dict value without a + ``key`` and the metric has no primary entry. """ violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): + limits = [bound[side] for side in ("min", "max") if side in bound] + if not all(_is_finite(limit) for limit in limits): + raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") row = report["metrics"].get(name) if row is None or row["status"] != "ok": violations.append(f"{name}: {'not reported' if row is None else row['status']}") diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 1071cb9..cb6e87c 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -43,10 +43,12 @@ def _log(tmp_path: Path, run_id: str, successes: list[bool]) -> None: (tmp_path / f"result.{run_id}.attempts.jsonl").write_text("\n".join(lines) + "\n") -def _run(tmp_path: Path, run_id: str) -> str: +def _run(tmp_path: Path, run_id: str, source_sha256: str | None = None) -> str: """A completed run: its result file and attempt log.""" result_file = tmp_path / "result.json" run = {"run_id": run_id, "attempts_file": f"result.{run_id}.attempts.jsonl"} + if source_sha256 is not None: + run["inputs"] = {"source": {"path": "source.hdf5", "sha256": source_sha256}} result_file.write_text(json.dumps({"num_success": 1, "num_attempts": 1, "run": run})) _log(tmp_path, run_id, [True]) return str(result_file) @@ -82,18 +84,23 @@ def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): assert "belongs to another run" in report["warnings"][0] -def test_report_refuses_records_whose_counts_disagree(tmp_path): +def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} with pytest.raises(ValueError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + dataset = _dataset(tmp_path / "d.hdf5", entry=entry) result_file = _run(tmp_path, "run-a") _log(tmp_path, "run-a", [True, True]) with pytest.raises(ValueError, match="attempt log holds 2 attempts"): - build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=result_file, metrics={}) + build_report(dataset, result_file=result_file, metrics={}) + result_file = _run(tmp_path, "run-a", source_sha256="0" * 64) + with pytest.raises(ValueError, match="not the one the run started from"): + build_report(dataset, result_file=result_file, source=_dataset(tmp_path / "source.hdf5"), metrics={}) -def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): + +def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} missing = {"status": "ok", "value": {"p50": 1.0}, "primary": "p95", "version": 1} report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}, "gap": missing}} @@ -105,6 +112,9 @@ def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): violations = gate_violations(report, str(gate)) assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] + gate.write_text("absent: {min: .nan}\n") + with pytest.raises(ValueError, match="not a finite number"): + gate_violations(report, str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 18a9ff8..9a933ad 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -22,7 +22,8 @@ Each metric reports one of three things: The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success -counts disagree, as they do after datasets are merged. +counts disagree, as they do after datasets are merged, and a ``--source`` other than the dataset the run +started from. Gating ------ @@ -35,7 +36,7 @@ Gating planning_time: {max: 5.0, key: p95} A gated metric also violates the gate if it is not applicable, fails, or reports a different -``version``. +``version``. A bound that is not a finite number is an error. Built-in metrics ---------------- From 3d4f4350a1276fafd3848c9d0db1c65867d843e1 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 22:51:05 -0500 Subject: [PATCH 15/24] fix: require scalar gate bounds and qualify the source check in the docs - A gate bound must be an int or float that is finite; YAML booleans and mappings are errors instead of passing or failing the comparison. - Docs: the source is verified only when the result file records it. --- autodata_metrics/report.py | 7 ++++++- autodata_tests/metrics/test_report.py | 7 ++++--- docs/pages/references/metrics.rst | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index d77d285..f8724aa 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -189,7 +189,7 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): limits = [bound[side] for side in ("min", "max") if side in bound] - if not all(_is_finite(limit) for limit in limits): + if not all(_is_bound(limit) for limit in limits): raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") row = report["metrics"].get(name) if row is None or row["status"] != "ok": @@ -212,6 +212,11 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: return violations +def _is_bound(limit: Any) -> bool: + """Whether a gate bound is a finite number; YAML reads ``true`` and ``false`` as bools, which are not bounds.""" + return isinstance(limit, int | float) and not isinstance(limit, bool) and math.isfinite(limit) + + def _format_value(value: Any) -> str: if isinstance(value, Mapping): return ", ".join(f"{key}={_format_value(item)}" for key, item in value.items()) or "none" diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index cb6e87c..d5a5b90 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -112,9 +112,10 @@ def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): violations = gate_violations(report, str(gate)) assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] - gate.write_text("absent: {min: .nan}\n") - with pytest.raises(ValueError, match="not a finite number"): - gate_violations(report, str(gate)) + for bad_bound in (".nan", "false", "{p95: 1}"): + gate.write_text(f"absent: {{min: {bad_bound}}}\n") + with pytest.raises(ValueError, match="not a finite number"): + gate_violations(report, str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 9a933ad..de230ad 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -22,8 +22,8 @@ Each metric reports one of three things: The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success -counts disagree, as they do after datasets are merged, and a ``--source`` other than the dataset the run -started from. +counts disagree, as they do after datasets are merged. When the result file records the run's source, the +report also refuses a different ``--source``; without that record, the source is not verified. Gating ------ From 573e0499f0410549d8dccf829dff6c5d95fdfc4c Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 09:01:45 -0500 Subject: [PATCH 16/24] fix: warn about a dataset without an autodata entry only against a run record A dataset reported without a result file has no run to match, so the episode-count-only warning applies only when a result is given. --- autodata_metrics/report.py | 4 +++- autodata_tests/metrics/test_report.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 229356c..1b58e2f 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -72,8 +72,10 @@ def check_same_run(evidence: Evidence) -> list[str]: f" {result['num_attempts']} and {result['num_success']}." ) entry = evidence.env_args.get("autodata") - if entry is None: + if entry is None and result is not None: return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] + if entry is None: + return [] if entry["exported_episodes"] != num_episodes: raise ValueError( f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index d5a5b90..8fd7a76 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -84,6 +84,13 @@ def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): assert "belongs to another run" in report["warnings"][0] +def test_dataset_without_an_entry_is_flagged_only_with_a_run_record(tmp_path): + dataset = _dataset(tmp_path / "d.hdf5") + + assert build_report(dataset, metrics={})["warnings"] == [] + assert "episode count only" in build_report(dataset, result_file=_run(tmp_path, "run-a"), metrics={})["warnings"][0] + + def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} with pytest.raises(ValueError, match="merged or edited"): From 1a591f2ea3cd78e51cfc1bc2c429d458a4403b98 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 09:55:32 -0500 Subject: [PATCH 17/24] feat: summarize the metrics report on the console and time annotation from process start - The console summary leads with a verdict (valid episodes, computed and not-computed metrics, the gate), lists the computed metrics by area with their source baseline and what their sample counts, and groups the metrics that do not apply by what would make them apply. With --gate, gated metrics are marked met or failed. - @metric declares what a metric's sample count counts. - Pass rates report which checks ran and which did not apply; valid_rotations no longer passes an episode that records no rotation. - gate_results returns the result of every gated metric. - Annotation records its mode and its setup time, timed from process start like generation. time_to_dataset version 2 counts annotation setup and keeps manual annotation time out of the total. - Docs: how to read the summary; the generate and validate workflow steps show how to record and report metrics. --- autodata_metrics/__init__.py | 2 +- autodata_metrics/evidence.py | 2 + autodata_metrics/registry.py | 4 + autodata_metrics/report.py | 217 ++++++++++++++---- autodata_metrics/validity.py | 20 +- autodata_metrics/viability.py | 27 ++- autodata_tests/e2e/test_annotation.py | 1 + autodata_tests/metrics/test_report.py | 33 ++- autodata_tests/metrics/test_validity.py | 4 + autodata_tests/metrics/test_viability.py | 19 +- docs/pages/concepts/data_generator.rst | 6 +- docs/pages/references/metrics.rst | 35 ++- .../step_3_generate_dataset.rst | 3 + .../step_4_validate_dataset.rst | 16 ++ scripts/annotate_demos.py | 6 +- scripts/validate_dataset.py | 9 +- 16 files changed, 324 insertions(+), 80 deletions(-) diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py index 64b9ee0..f75f7e1 100644 --- a/autodata_metrics/__init__.py +++ b/autodata_metrics/__init__.py @@ -20,4 +20,4 @@ check, metric, ) -from autodata_metrics.report import build_report, format_report, gate_violations +from autodata_metrics.report import build_report, format_report, gate_results diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index e6c3bcb..274ec44 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -62,6 +62,7 @@ class Evidence: episodes: The episodes that passed every episode check; dataset metrics read these. failures: Per episode check that ran, the failing episodes' names and reasons. check_groups: Per episode check that ran, its group. + skipped_checks: Per episode check that did not apply to any episode, its group. env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer. result: Generation result, or None for a dataset without a run record or an interrupted run. run: The result's ``run`` block, or empty. @@ -81,6 +82,7 @@ def __init__( self.episodes = list(self.all_episodes) self.failures: dict[str, dict[str, str]] = {} self.check_groups: dict[str, str] = {} + self.skipped_checks: dict[str, str] = {} self.result = result self.run: Mapping[str, Any] = (result or {}).get("run", {}) self.attempts = attempts diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py index 433c687..c2be456 100644 --- a/autodata_metrics/registry.py +++ b/autodata_metrics/registry.py @@ -49,6 +49,7 @@ class MetricSpec: signal: str scope: Literal["episode", "dataset", "run"] unit: str + sample: str better: Literal["higher", "lower"] | None requires: tuple[str, ...] primary: str | None @@ -78,6 +79,7 @@ def metric( signal: str, scope: Literal["episode", "dataset", "run"], unit: str, + sample: str, better: Literal["higher", "lower"] | None = None, requires: tuple[str, ...] | list[str] = (), primary: str | None = None, @@ -93,6 +95,7 @@ def metric( scope: ``"episode"`` metrics summarize the episode checks over every episode; ``"dataset"`` metrics read the episodes that passed them; ``"run"`` metrics read attempts and the run record. unit: Unit of the value. + sample: What the value's sample count counts, in the plural, e.g. ``"attempts"`` or ``"episodes"``. better: Which direction is better, if either. requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`. primary: For a dict value, the entry a gate checks by default. @@ -110,6 +113,7 @@ def register(compute: Callable[[Evidence], Value]) -> Callable[[Evidence], Value signal=signal, scope=scope, unit=unit, + sample=sample, better=better, requires=tuple(requires), primary=primary, diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 1b58e2f..063c3ce 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -10,6 +10,8 @@ import json import math import numbers +import re +import textwrap import yaml from collections.abc import Mapping from pathlib import Path @@ -73,7 +75,9 @@ def check_same_run(evidence: Evidence) -> list[str]: ) entry = evidence.env_args.get("autodata") if entry is None and result is not None: - return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] + return [ + "the dataset was recorded without --record_metric_inputs, so it is matched to its run by episode count only" + ] if entry is None: return [] if entry["exported_episodes"] != num_episodes: @@ -93,6 +97,7 @@ def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: """Run the episode checks and keep only the episodes that pass all of them in ``evidence.episodes``.""" for spec in checks.values(): if any(not evidence.has(key) for key in spec.requires): + evidence.skipped_checks[spec.name] = spec.group continue failed: dict[str, str] = {} applied = False @@ -109,6 +114,8 @@ def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: if applied: evidence.failures[spec.name] = failed evidence.check_groups[spec.name] = spec.group + else: + evidence.skipped_checks[spec.name] = spec.group failing = {name for failed in evidence.failures.values() for name in failed} evidence.episodes = [episode for episode in evidence.all_episodes if episode.name not in failing] @@ -120,6 +127,7 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: "signal": spec.signal, "scope": spec.scope, "unit": spec.unit, + "sample": spec.sample, "better": spec.better, "primary": spec.primary, "version": spec.version, @@ -128,7 +136,7 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": "no valid episodes"} missing = next((key for key in spec.requires if not evidence.has(key)), None) if missing is not None: - return {**row, "status": "not_applicable", "reason": f"missing {missing}"} + return {**row, "status": "not_applicable", "reason": f"missing {missing}", "missing": missing} try: value = spec.compute(evidence) except NotApplicable as exc: @@ -218,41 +226,42 @@ def build_report( } -def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: - """Bounds in ``gate_file`` that the report violates. +def gate_results(report: Mapping[str, Any], gate_file: str) -> dict[str, str | None]: + """Per metric in ``gate_file``, None when the report meets its bounds, else why it does not. The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or - at another version is a violation. + at another version fails. Raises: ValueError: If a gate entry has a bound that is not a finite number, or bounds a dict value without a ``key`` and the metric has no primary entry. """ - violations = [] + results = {} for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): limits = [bound[side] for side in ("min", "max") if side in bound] if not all(_is_bound(limit) for limit in limits): raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") - row = report["metrics"].get(name) - if row is None or row["status"] != "ok": - violations.append(f"{name}: {'not reported' if row is None else row['status']}") - continue - if "version" in bound and bound["version"] != row["version"]: - violations.append(f"{name}: version {row['version']}, gate written for version {bound['version']}") - continue - key = bound.get("key", row["primary"]) - value = row["value"].get(key) if key else row["value"] - if isinstance(value, Mapping): - raise ValueError(f"Gate entry {name!r} needs a key into {sorted(value)}.") - if not _is_finite(value): - violations.append(f"{name}: {value!r} is not a finite number") - continue - if "min" in bound and value < bound["min"]: - violations.append(f"{name}: {value:.4g} is below {bound['min']}") - if "max" in bound and value > bound["max"]: - violations.append(f"{name}: {value:.4g} is above {bound['max']}") - return violations + results[name] = _gate_failure(name, report["metrics"].get(name), bound) + return results + + +def _gate_failure(name: str, row: Mapping[str, Any] | None, bound: Mapping[str, Any]) -> str | None: + if row is None or row["status"] != "ok": + return "not reported" if row is None else row["status"] + if "version" in bound and bound["version"] != row["version"]: + return f"version {row['version']}, gate written for version {bound['version']}" + key = bound.get("key", row["primary"]) + value = row["value"].get(key) if key else row["value"] + if isinstance(value, Mapping): + raise ValueError(f"Gate entry {name!r} needs a key into {sorted(value)}.") + if not _is_finite(value): + return f"{value!r} is not a finite number" + if "min" in bound and value < bound["min"]: + return f"{value:.4g} is below {bound['min']}" + if "max" in bound and value > bound["max"]: + return f"{value:.4g} is above {bound['max']}" + return None def _is_bound(limit: Any) -> bool: @@ -260,27 +269,143 @@ def _is_bound(limit: Any) -> bool: return isinstance(limit, int | float) and not isinstance(limit, bool) and math.isfinite(limit) -def _format_value(value: Any) -> str: - if isinstance(value, Mapping): - return ", ".join(f"{key}={_format_value(item)}" for key, item in value.items()) or "none" - return f"{value:.3g}" if isinstance(value, float) else str(value) +CONSOLE_WIDTH = 100 +"""Width the console summary wraps its lists of metrics to.""" -def format_report(report: Mapping[str, Any]) -> str: - """The report as an aligned text table, grouped by area.""" - rows = [("Metric", "Area", "Value", "Unit", "n")] - for name, row in sorted(report["metrics"].items(), key=lambda item: (AREAS.index(item[1]["area"]), item[0])): - if row["status"] == "ok": - rows.append((name, row["area"], _format_value(row["value"]), row["unit"], str(row["n"]))) - else: - rows.append((name, row["area"], f"{row['status']}: {row['reason']}", "", "")) - widths = [max(len(row[column]) for row in rows) for column in range(len(rows[0]))] - lines = [" ".join(cell.ljust(width) for cell, width in zip(row, widths)).rstrip() for row in rows] - lines.insert(1, " ".join("-" * width for width in widths)) - failed = [f"{name}={entry['failed']}" for name, entry in report["check_failures"].items() if entry["failed"]] - lines.append( - f"episodes: {report['num_valid_episodes']} valid of {report['num_episodes']}" - + (f"; failed checks: {', '.join(failed)}" if failed else "") - ) - lines += [f"warning: {warning}" for warning in report["warnings"]] +def format_report(report: Mapping[str, Any], gate: Mapping[str, str | None] | None = None) -> str: + """The report as a console summary: a verdict, the computed metrics by area, and what was not computed and why. + + The JSON report holds every detail; the summary keeps a metric's primary value, its source baseline, its sample + count and, for rates, the 95% interval. + + Args: + report: A report from :func:`build_report`. + gate: Results from :func:`gate_results`, to mark each gated metric as met or failed. + """ + inputs, metrics = report["inputs"], report["metrics"] + computed = {name: row for name, row in metrics.items() if row["status"] == "ok"} + errors = {name: row for name, row in metrics.items() if row["status"] == "error"} + skipped = {name: row for name, row in metrics.items() if row["status"] == "not_applicable"} + + title = [f"AutoData metrics · {Path(inputs['dataset']['path']).name}"] + title += [f"run {inputs['run_id'][:8]}"] if inputs["run_id"] else [] + title += [f"source {Path(inputs['source']['path']).name}"] if inputs["source"] else [] + verdict = [f"{report['num_valid_episodes']}/{report['num_episodes']} episodes valid", f"{len(computed)} computed"] + verdict += [f"{len(skipped)} not computed"] if skipped else [] + verdict += [_count(len(errors), "errors")] if errors else [] + if gate is not None: + failed = sum(reason is not None for reason in gate.values()) + verdict.append( + f"gate failed on {failed} of {len(gate)}" if failed else f"gate passed on {len(gate)} of {len(gate)}" + ) + lines = [" · ".join(title), " · ".join(verdict)] + for name, entry in report["check_failures"].items(): + if entry["failed"]: + more = ", …" if entry["failed"] > len(entry["examples"]) else "" + lines.append( + f" ✗ {name} failed {_count(entry['failed'], 'episodes')}: {', '.join(entry['examples'])}{more}" + ) + + rows = [] + for name, row in computed.items(): + mark = " " if gate is None or name not in gate else ("✗" if gate[name] else "✓") + baseline = row.get("baseline") + source = "" if baseline is None else _console_value(baseline, row, labelled=False) + rows.append((row["area"], mark, name, _console_value(row["value"], row), source, _based_on(row))) + if rows: + has_source = any(row[4] for row in rows) + name_width = max(len(row[2]) for row in rows) + value_width = max(len(row[3]) for row in rows + [("", "", "", "Value")]) + source_width = max(len(row[4]) for row in rows + [("", "", "", "", "Source")]) + + def line(mark: str, name: str, value: str, source: str, based_on: str) -> str: + cells = [f"{mark} {name.ljust(name_width)}", value.rjust(value_width)] + cells += [source.rjust(source_width)] if has_source else [] + return " ".join(cells + [based_on]).rstrip() + + lines += ["", line(" ", "", "Value", "Source", "Based on")] + for area in AREAS: + area_rows = sorted((row for row in rows if row[0] == area), key=lambda row: row[2]) + if area_rows: + lines.append(area.upper()) + lines += [line(*row[1:]) for row in area_rows] + if errors: + lines += ["", "ERRORS"] + [f" {name}: {row['reason']}" for name, row in errors.items()] + if skipped: + groups: dict[str, list[str]] = {} + for name, row in skipped.items(): + groups.setdefault(_not_computed_because(row, inputs), []).append(name) + lines += ["", "NOT COMPUTED"] + for reason, names in sorted(groups.items(), key=lambda item: -len(item[1])): + lines += textwrap.wrap( + ", ".join(sorted(names)), CONSOLE_WIDTH, initial_indent=f" {reason}: ", subsequent_indent=" " + ) + if report["warnings"]: + lines += [""] + [f"Warning: {warning}" for warning in report["warnings"]] return "\n".join(lines) + + +def _count(count: int, noun: str) -> str: + return f"{count:,} {noun.removesuffix('s') if count == 1 else noun}" + + +def _based_on(row: Mapping[str, Any]) -> str: + """The sample count, with a rate's 95% interval and a pass rate's check coverage.""" + parts = [_count(row["n"], row["sample"])] + interval = row["details"].get("interval_95") + if interval: + low, high = (_console_number(bound, row["unit"]) for bound in interval) + parts.append(f"95% CI {low.removesuffix('%')}–{high}") + if row["details"].get("checks_skipped"): + run, skipped = len(row["details"]["checks_run"]), len(row["details"]["checks_skipped"]) + parts.append(f"{run} of {run + skipped} checks") + return " · ".join(parts) + + +def _console_value(value: Any, row: Mapping[str, Any], labelled: bool = True) -> str: + """A value, or a dict value's primary entry (every entry without one), as the console shows it.""" + if not isinstance(value, Mapping): + return _console_number(value, row["unit"]) + keys = [row["primary"]] if row["primary"] else list(value) + return ", ".join((f"{key} " if labelled else "") + _console_number(value[key], row["unit"]) for key in keys) + + +def _console_number(value: float, unit: str) -> str: + """Fractions as percentages, seconds as durations; other units follow the number unless they hold a space, which + marks a description of the quantity rather than a unit.""" + if unit.startswith("fraction"): + return f"{100 * value:.3g}%" + if unit == "s" or unit.startswith("s "): + return _duration(value) + text = f"{value:,.0f}" if abs(value) >= 100 else re.sub(r"e\+?(-?)0*(\d)", r"e\1\2", f"{value:.3g}") + return text if " " in unit else f"{text} {unit}" + + +def _duration(seconds: float) -> str: + if seconds < 120: + return f"{seconds:.3g} s" + minutes, rest = divmod(round(seconds), 60) + if minutes < 120: + return f"{minutes} min {rest:02d} s" + return f"{minutes // 60} h {minutes % 60:02d} min" + + +_METRIC_INPUT_KEYS = ("env_args.autodata", "step_success", "obs/datagen_info/") +"""Evidence keys a generated dataset holds only when recorded with ``--record_metric_inputs``.""" + + +def _not_computed_because(row: Mapping[str, Any], inputs: Mapping[str, Any]) -> str: + """What would make a not-applicable metric computable, or its own reason.""" + missing = row.get("missing") + if missing is None: + return row["reason"] + if missing.startswith("attempts.extras."): + return "not reported by this algorithm" + if missing.partition(".")[0] == "source": + return "needs --source" if inputs["source"] is None else "needs a --source recorded with --record_metric_inputs" + if missing == "attempts" or missing.startswith("run."): + return "needs --result_file" if inputs["result_file"] is None else "needs --record_metric_inputs" + if missing.startswith(_METRIC_INPUT_KEYS): + return "needs --record_metric_inputs" + return row["reason"] diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 0751aef..1995cd4 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -71,7 +71,7 @@ def valid_rotations(episode: Episode, ev: Evidence) -> str | None: Both tests hold for either quaternion order, so the check needs no convention. """ - bad = [] + bad, checked = [], False for path, dataset in episode.datasets(): if path.startswith(OBSERVED_POSE_PATHS) and dataset.shape[-2:] == (4, 4): error = _rotation_errors(dataset[()]).max(initial=0.0) @@ -79,8 +79,11 @@ def valid_rotations(episode: Episode, ev: Evidence) -> str | None: error = np.abs(np.linalg.norm(dataset[()][..., 3:7], axis=-1) - 1.0).max(initial=0.0) else: continue + checked = True if error > ROTATION_TOLERANCE: bad.append(path) + if not checked: + raise NotApplicable("no recorded rotations") return f"invalid rotations in {', '.join(bad)}" if bad else None @@ -132,14 +135,15 @@ def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: return None -def _pass_rate(ev: Evidence, group: str) -> Value: +def _pass_rate(ev: Evidence, group: str, none_apply: str) -> Value: names = [name for name, check_group in ev.check_groups.items() if check_group == group] if not names: - raise NotApplicable(f"no {group} check applies to this dataset") + raise NotApplicable(none_apply) if not ev.all_episodes: raise NotApplicable("the dataset has no episodes") failing = {episode for name in names for episode in ev.failures[name]} - details = {f"{name}_failed": len(ev.failures[name]) for name in names} + skipped = [name for name, check_group in ev.skipped_checks.items() if check_group == group] + details = {"checks_run": names, "checks_skipped": skipped} return Value(1 - len(failing) / len(ev.all_episodes), n=len(ev.all_episodes), details=details) @@ -149,11 +153,12 @@ def _pass_rate(ev: Evidence, group: str) -> Value: signal="data correctness", scope="episode", unit="fraction", + sample="episodes", better="higher", ) def correctness_pass_rate(ev: Evidence) -> Value: """Share of episodes passing every correctness check: finite, aligned, valid rotations, actions in bounds.""" - return _pass_rate(ev, "correctness") + return _pass_rate(ev, "correctness", "no correctness check applies to this dataset") @metric( @@ -162,11 +167,12 @@ def correctness_pass_rate(ev: Evidence) -> Value: signal="data correctness", scope="episode", unit="fraction", + sample="episodes", better="higher", ) def image_stream_pass_rate(ev: Evidence) -> Value: """Share of episodes whose camera streams hold one frame per step, no black frame and no frozen run.""" - return _pass_rate(ev, "image") + return _pass_rate(ev, "image", "no camera streams") @metric( @@ -175,6 +181,7 @@ def image_stream_pass_rate(ev: Evidence) -> Value: signal="task correctness", scope="dataset", unit="fraction", + sample="episodes", better="higher", requires=["step_success"], baseline=True, @@ -191,6 +198,7 @@ def final_success_rate(ev: Evidence) -> Value: signal="data correctness", scope="dataset", unit="max |R^T R - I|", + sample="poses", better="lower", primary="p95", requires=["obs/datagen_info/target_eef_pose"], diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index a8de7a1..d5a2bbc 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -58,6 +58,7 @@ def _counted_attempts(ev: Evidence) -> tuple[list[dict[str, Any]], bool]: signal="generation performance", scope="run", unit="fraction", + sample="attempts", better="higher", requires=["attempts"], ) @@ -77,6 +78,7 @@ def generation_success_rate(ev: Evidence) -> Value: signal="generation performance", scope="run", unit="demos/GPU-h", + sample="demos", better="higher", requires=["run.generation_time_s", "run.num_gpus", "env_args.autodata.exported_episodes"], ) @@ -96,19 +98,25 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: signal="time to dataset", scope="run", unit="s", + sample="runs", better="lower", requires=["run.setup_time_s", "run.generation_time_s"], + version=2, ) def time_to_dataset(ev: Evidence) -> Value: - """Wall time of the automated steps: annotation of the source, setup and generation.""" + """Wall time of the automated steps, each process timed from its start: annotating the source and generating.""" annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} - parts = { - "annotation_s": annotation.get("time_s"), - "setup_s": ev.run["setup_time_s"], - "generation_s": ev.run["generation_time_s"], - } - details = {**parts, "note": "automated steps only; manual recording and annotation time is not measured"} - return Value(sum(seconds for seconds in parts.values() if seconds is not None), n=1, details=details) + automated = {"setup_s": ev.run["setup_time_s"], "generation_s": ev.run["generation_time_s"]} + manual = {} + if annotation: + # An entry written before the mode was recorded counts as automatic, as in version 1. + if annotation.get("mode", "auto") == "auto": + automated["annotation_s"] = annotation.get("setup_time_s", 0.0) + annotation["time_s"] + else: + automated["annotation_s"] = annotation["setup_time_s"] + manual["manual_annotation_s"] = annotation["time_s"] + details = {**automated, **manual, "note": "manual recording and manual annotation are not in the total"} + return Value(sum(automated.values()), n=1, details=details) @metric( @@ -117,6 +125,7 @@ def time_to_dataset(ev: Evidence) -> Value: signal="time to dataset", scope="run", unit="fraction", + sample="recordings", better="higher", requires=["source.env_args.autodata.annotation"], ) @@ -135,6 +144,7 @@ def annotation_pass_rate(ev: Evidence) -> Value: signal="failure causes", scope="run", unit="fraction of attempts", + sample="attempts", better="lower", requires=["attempts.extras.planning_failed"], ) @@ -151,6 +161,7 @@ def planning_failure_rate(ev: Evidence) -> Value: signal="generation performance", scope="run", unit="s per attempt", + sample="attempts", better="lower", primary="p95", requires=["attempts.extras.planning_time_s"], diff --git a/autodata_tests/e2e/test_annotation.py b/autodata_tests/e2e/test_annotation.py index fa93c2e..f8e8991 100644 --- a/autodata_tests/e2e/test_annotation.py +++ b/autodata_tests/e2e/test_annotation.py @@ -41,6 +41,7 @@ def test_automatic_annotation_from_dataset(tmp_path): assert_valid_dataset(str(output_file), min_num_demos=1) entry = assert_recorded_episodes(str(output_file)) assert entry["annotation"]["annotated_episodes"] == entry["exported_episodes"] + assert entry["annotation"]["mode"] == "auto" and entry["annotation"]["setup_time_s"] > 0 with h5py.File(input_file, "r") as source, h5py.File(output_file, "r") as annotated: assert len(annotated["data"]) == len(source["data"]) assert json.loads(annotated["data"].attrs["env_args"])["env_name"] == "IsaacContrib-Stack-Cube-Franka-IK-Rel" diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 8fd7a76..d6fd750 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -1,17 +1,18 @@ # Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests of how the report runs metrics, refuses mismatched runs and gates values.""" +"""Tests of how the report runs metrics, refuses mismatched runs, gates values and summarizes them.""" import h5py import json +import re import subprocess import sys from pathlib import Path import pytest -from autodata_metrics import MetricSpec, Value, build_report, gate_violations +from autodata_metrics import MetricSpec, Value, build_report, format_report, gate_results def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: @@ -22,6 +23,7 @@ def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: signal="test", scope="run", unit="fraction", + sample="samples", better="higher", requires=requires, primary=None, @@ -116,13 +118,14 @@ def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): "low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\ngap: {max: 2}\n" ) - violations = gate_violations(report, str(gate)) + results = gate_results(report, str(gate)) - assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] + assert [name for name, reason in results.items() if reason is not None] == ["low", "old", "absent", "gap"] + assert results["fine"] is None for bad_bound in (".nan", "false", "{p95: 1}"): gate.write_text(f"absent: {{min: {bad_bound}}}\n") with pytest.raises(ValueError, match="not a finite number"): - gate_violations(report, str(gate)) + gate_results(report, str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): @@ -135,10 +138,28 @@ def test_non_finite_numbers_never_reach_the_report(tmp_path): gate.write_text("m: {min: 0}\n") assert report["metrics"]["m"]["status"] == "error" - assert gate_violations(report, str(gate)) == ["m: error"] + assert gate_results(report, str(gate)) == {"m": "error"} assert report["metrics"]["d"]["details"] == {"interval_95": [0.5, None]} +def test_console_summary_marks_the_gate_and_says_how_to_compute_the_rest(tmp_path): + metrics = { + "rate": _spec("rate", lambda ev: Value(0.25, n=4)), + "held": _spec("held", lambda ev: Value(1.0, n=1), ("step_success",)), + "planned": _spec("planned", lambda ev: Value(1.0, n=1), ("attempts.extras.planning_time_s",)), + } + report = build_report(_dataset(tmp_path / "d.hdf5"), metrics=metrics) + gate = tmp_path / "gate.yaml" + gate.write_text("rate: {min: 0.5}\n") + + summary = format_report(report, gate_results(report, str(gate))) + + assert "gate failed on 1 of 1" in summary + assert re.search(r"✗ rate +25% +4 samples", summary) + assert "needs --record_metric_inputs: held" in summary + assert "not reported by this algorithm: planned" in summary + + def test_metrics_import_without_torch_or_isaaclab(): code = "import sys, autodata_metrics; print(sorted({'torch', 'isaaclab', 'autodata_core'} & set(sys.modules)))" output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index a252907..655640c 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -42,6 +42,10 @@ def test_failing_episodes_are_counted_and_left_out_of_dataset_metrics(tmp_path): ) assert report["metrics"]["correctness_pass_rate"]["value"] == pytest.approx(1 / 3) + assert report["metrics"]["correctness_pass_rate"]["details"]["checks_skipped"] == [ + "valid_rotations", + "pose_action_bounds", + ] assert report["metrics"]["final_success_rate"]["value"] == 1.0 and report["metrics"]["final_success_rate"]["n"] == 1 assert {name: entry["failed"] for name, entry in report["check_failures"].items() if entry["failed"]} == { "finite": 1, diff --git a/autodata_tests/metrics/test_viability.py b/autodata_tests/metrics/test_viability.py index ecaa78e..f8fe5ac 100644 --- a/autodata_tests/metrics/test_viability.py +++ b/autodata_tests/metrics/test_viability.py @@ -3,9 +3,11 @@ """Known-answer tests of the viability statistics.""" +from types import SimpleNamespace + import pytest -from autodata_metrics.viability import attempts_before_stop, wilson_interval +from autodata_metrics.viability import attempts_before_stop, time_to_dataset, wilson_interval def test_wilson_interval_matches_known_values(): @@ -26,3 +28,18 @@ def test_attempts_before_stop_keeps_attempts_started_before_the_first_env_finish assert len(kept) == 11 and sum(attempt["success"] for attempt in kept) == 3 assert attempts_before_stop(long_successes[:2] + short_failures, num_envs=2) is None assert attempts_before_stop(short_failures, num_envs=2) is None + + +def test_time_to_dataset_counts_automated_steps_and_keeps_manual_annotation_apart(): + run = {"setup_time_s": 20.0, "generation_time_s": 80.0} + + def measured(annotation: dict): + return time_to_dataset( + SimpleNamespace(source=SimpleNamespace(env_args={"autodata": {"annotation": annotation}}), run=run) + ) + + automatic = measured({"mode": "auto", "setup_time_s": 15.0, "time_s": 60.0}) + manual = measured({"mode": "manual", "setup_time_s": 15.0, "time_s": 600.0}) + + assert automatic.value == 175.0 + assert (manual.value, manual.details["manual_annotation_s"]) == (115.0, 600.0) diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 244451b..525100d 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -172,8 +172,10 @@ The ``run`` block is written only with ``--record_metric_inputs``. With that opt dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, and the -embodiment's optional datasheet ``hardware_limits``. For a dataset generated with ``--result_file``, -the ``run_id`` ties it to that run's attempt log and, once the run completes, its result file. +embodiment's optional datasheet ``hardware_limits``. An annotated dataset's entry also records the +annotation mode (automatic or manual), its episode counts, and its setup and annotation times. For a +dataset generated with ``--result_file``, the ``run_id`` ties it to that run's attempt log and, once the +run completes, its result file. The GenerationAlgorithm Plug-In ------------------------------- diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index f108485..d5ed482 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -13,11 +13,35 @@ without it, most metrics report that their inputs are missing. --source datasets/annotated_datasets/dataset_franka_annotated.hdf5 \ --report_file datasets/generated_report.json -Each metric reports one of three things: - -- a value, with its unit and sample count; -- ``not_applicable``, with the reason, such as a missing recorded input; -- ``error``, when it fails. The other metrics are still computed. +The command prints a summary; ``--report_file`` writes the full report, with every detail, as JSON. + +.. code-block:: text + + AutoData metrics · gr1.hdf5 · run e79e37b1 · source dataset_gr1_annotated.hdf5 + 6/6 episodes valid · 7 computed · 3 not computed · gate failed on 2 of 4 + + Value Source Based on + VALIDITY + ✗ action_rotation_error p95 0.0345 3.58e-7 1,612 poses + ✓ correctness_pass_rate 100% 6 episodes + ✓ final_success_rate 100% 6 episodes + image_stream_pass_rate 100% 6 episodes + VIABILITY + demos_per_gpu_hour 65.8 demos/GPU-h 6 demos + ✗ generation_success_rate 21.4% 28 attempts · 95% CI 10.2–39.5% + time_to_dataset 6 min 11 s 1 run + + NOT COMPUTED + not reported by this algorithm: planning_failure_rate, planning_time + needs a --source recorded with --record_metric_inputs: annotation_pass_rate + +- The second line is the verdict: valid episodes, how many metrics were computed, and the gate. +- **Value** is the metric's value; a metric with percentiles shows the one a gate checks by default. + **Source** is the same metric on the ``--source`` dataset. **Based on** is the sample count, with a + rate's 95% interval and, when some checks did not apply, how many checks a pass rate rests on. +- With ``--gate``, ✓ and ✗ mark the gated metrics. +- **Not computed** groups the metrics that do not apply by what would make them apply. +- A metric that fails is listed under **Errors**; the other metrics are still computed. The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own @@ -69,6 +93,7 @@ requires. Without that evidence, the report says which key is missing and does n signal="generation performance", scope="run", unit="s", + sample="attempts", better="lower", requires=["attempts.extras.planning_time_s"], ) diff --git a/docs/pages/workflows/franka_cube_stack_mimicgen/step_3_generate_dataset.rst b/docs/pages/workflows/franka_cube_stack_mimicgen/step_3_generate_dataset.rst index 0e15b62..04734e4 100644 --- a/docs/pages/workflows/franka_cube_stack_mimicgen/step_3_generate_dataset.rst +++ b/docs/pages/workflows/franka_cube_stack_mimicgen/step_3_generate_dataset.rst @@ -71,6 +71,9 @@ For dataset-scale generation, run headless (``--viz none``) with parallel enviro Progress is printed after every attempt (successful demonstrations, attempts, and the running success rate). +To report the run's :doc:`metrics <../../references/metrics>` in the next step, add +``--record_metric_inputs`` and ``--result_file ./datasets/generated_dataset_mimicgen_franka.json``. + .. note:: **Upgrade validation:** the 100-environment GPU benchmark achieved 38.0% success diff --git a/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst b/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst index ea1a7d4..79f50b2 100644 --- a/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst +++ b/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst @@ -17,6 +17,22 @@ checked for the required fields (``actions``, ``initial_state``, ``obs``): For unreadable, invalid, or truncated files, see :ref:`troubleshooting-hdf5`. +Metrics +^^^^^^^ + +For a dataset generated with ``--record_metric_inputs`` and ``--result_file``, report its metrics: + +.. code-block:: bash + + python scripts/validate_dataset.py ./datasets/generated_dataset_mimicgen_franka.hdf5 \ + --report \ + --result_file ./datasets/generated_dataset_mimicgen_franka.json \ + --source ./datasets/annotated_datasets/dataset_franka_annotated.hdf5 + +The summary lists each metric's value, the metrics that do not apply and what would make them apply. +See :doc:`../../references/metrics` for what each metric means and how to gate on it. + + Visual Validation ^^^^^^^^^^^^^^^^^ diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index ee5434f..e9ba4d6 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -30,11 +30,14 @@ """Launch Isaac Sim Simulator first.""" import argparse +import time from isaaclab.app import AppLauncher from autodata_utils.cli import callable_path +PROCESS_START = time.monotonic() + parser = argparse.ArgumentParser( description="Manually annotate AutoData source demonstrations with subtask signals.", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -109,7 +112,6 @@ import math # noqa: E402 import os # noqa: E402 import sys # noqa: E402 -import time # noqa: E402 import torch # noqa: E402 import traceback # noqa: E402 import uuid # noqa: E402 @@ -337,9 +339,11 @@ def main() -> int: ) if autodata_entry is not None: autodata_entry["annotation"] = { + "mode": "auto" if args_cli.auto else "manual", "input_episodes": len(episode_names), "processed_episodes": processed_episode_count, "annotated_episodes": exported_episode_count, + "setup_time_s": annotation_start - PROCESS_START, "time_s": time.monotonic() - annotation_start, } print("Exiting the app.") diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index 8b9bb43..440da73 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -23,7 +23,7 @@ import os import sys -from autodata_metrics import build_report, format_report, gate_violations +from autodata_metrics import build_report, format_report, gate_results REQUIRED_DEMO_FIELDS = { "actions": h5py.Dataset, @@ -196,14 +196,15 @@ def main(argv: list[str] | None = None) -> int: for module in args.metrics: importlib.import_module(module) report = build_report(args.files[0], result_file=args.result_file, source=args.source) + gate = gate_results(report, args.gate) if args.gate else None print() - print(format_report(report)) + print(format_report(report, gate)) if args.report_file: with open(args.report_file, "w", encoding="utf-8") as report_handle: json.dump(report, report_handle, indent=2, default=_json_value) report_handle.write("\n") - if args.gate: - violations = gate_violations(report, args.gate) + if gate is not None: + violations = [f"{name}: {reason}" for name, reason in gate.items() if reason is not None] print("\nGate passed." if not violations else "\nGate failed:\n " + "\n ".join(violations)) return 1 if num_invalid or violations else 0 From 19cd6607ea3052e57a9fae86aed6b0dc25684592 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 10:22:21 -0500 Subject: [PATCH 18/24] fix: tighten the episode checks and require valid episodes in the e2e tests - one_frame_per_step fails an episode that lacks a camera stream other episodes record. - finite reads floating-point arrays in chunks instead of whole. - aligned_lengths fails an episode with no actions. - valid_rotations checks every observed pose, all-zero ones included; only commanded targets skip all-zero placeholders. - A rotation's error includes |det R - 1|, so reflections fail. - assert_ok_metrics asserts that every episode passes the checks. --- autodata_metrics/evidence.py | 11 +++++ autodata_metrics/validity.py | 66 ++++++++++++++----------- autodata_tests/metrics/test_validity.py | 24 ++++++--- autodata_tests/utils/utils.py | 9 +++- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index e6c3bcb..280e676 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -9,6 +9,7 @@ import json import numpy as np from collections.abc import Mapping +from functools import cached_property from typing import Any @@ -106,3 +107,13 @@ def has(self, key: str) -> bool: if head == "run": return _has_path(self.run, rest) return bool(self.episodes) and all(key in episode for episode in self.episodes) + + @cached_property + def camera_paths(self) -> frozenset[str]: + """Paths of the camera streams, 4-D ``uint8`` arrays, that any episode records.""" + return frozenset( + path + for episode in self.all_episodes + for path, dataset in episode.datasets() + if dataset.dtype == np.uint8 and dataset.ndim == 4 + ) diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 0751aef..3e481fd 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -12,62 +12,64 @@ from autodata_metrics.registry import NotApplicable, Value, check, metric ROTATION_TOLERANCE = 1e-3 -"""Largest deviation from an orthonormal rotation, or from a unit quaternion, a recorded pose may show.""" +"""Largest deviation from a proper rotation, or from a unit quaternion, a recorded pose may show.""" BLACK_FRAME_MEAN = 2.0 """Mean pixel value, out of 255, below which a camera frame counts as black.""" FROZEN_FRAMES = 10 """Identical consecutive camera frames that count as a frozen stream.""" FRAME_CHUNK = 64 -"""Camera frames read at a time, so image checks stream instead of loading whole episodes.""" +"""Steps read at a time, so checks of camera frames and other large arrays stream instead of loading whole episodes.""" PER_EPISODE_PATHS = ("initial_state", "obs/datagen_info/subtask_term_signals", "obs/datagen_info/subtask_start_signals") """Paths whose arrays are not one entry per step: the initial state, and annotation's subtask signals.""" OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") """Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" -def _is_image(dataset: h5py.Dataset) -> bool: - return dataset.dtype == np.uint8 and dataset.ndim == 4 +def _image_streams(episode: Episode, ev: Evidence) -> dict[str, h5py.Dataset]: + """The episode's camera streams; not applicable to a dataset that records none.""" + if not ev.camera_paths: + raise NotApplicable("no camera streams") + return {path: dataset for path, dataset in episode.datasets() if path in ev.camera_paths} -def _image_streams(episode: Episode) -> list[tuple[str, h5py.Dataset]]: - streams = [(path, dataset) for path, dataset in episode.datasets() if _is_image(dataset)] - if not streams: - raise NotApplicable("no camera streams") - return streams +def _all_finite(dataset: h5py.Dataset) -> bool: + if dataset.ndim == 0: + return bool(np.isfinite(dataset[()])) + return all(np.isfinite(dataset[start : start + FRAME_CHUNK]).all() for start in range(0, len(dataset), FRAME_CHUNK)) @check("finite") def finite(episode: Episode, ev: Evidence) -> str | None: """Fails an episode with a NaN or infinite value in any floating-point array.""" - bad = [ - path for path, dataset in episode.datasets() if dataset.dtype.kind == "f" and not np.isfinite(dataset[()]).all() - ] + bad = [path for path, dataset in episode.datasets() if dataset.dtype.kind == "f" and not _all_finite(dataset)] return f"non-finite values in {', '.join(bad)}" if bad else None @check("aligned_lengths") def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: """Fails an episode whose per-step arrays do not all hold one entry per action; camera streams are checked apart.""" - if "actions" not in episode: + num_steps = len(episode["actions"]) if "actions" in episode else 0 + if num_steps == 0: return "no actions" - num_steps = len(episode["actions"]) bad = [ path for path, dataset in episode.datasets() - if not path.startswith(PER_EPISODE_PATHS) and not _is_image(dataset) and len(dataset) != num_steps + if not path.startswith(PER_EPISODE_PATHS) and path not in ev.camera_paths and len(dataset) != num_steps ] return f"{', '.join(bad)} do not hold {num_steps} steps" if bad else None def _rotation_errors(poses: np.ndarray) -> np.ndarray: - """Per pose, the largest entry of ``|R^T R - I|``; all-zero poses, a channel's placeholder target, are skipped.""" - rotations = poses[np.abs(poses).reshape(len(poses), -1).max(axis=1) > 0][..., :3, :3] - return np.abs(np.einsum("...ji,...jk->...ik", rotations, rotations) - np.eye(3)).max(axis=(-2, -1)) + """Per 4x4 pose, how far its rotation is from a proper one: the larger of the largest entry of ``|R^T R - I|`` + and ``|det R - 1|``, which also catches reflections.""" + rotations = poses[..., :3, :3] + orthonormality = np.abs(np.einsum("...ji,...jk->...ik", rotations, rotations) - np.eye(3)).max(axis=(-2, -1)) + return np.maximum(orthonormality, np.abs(np.linalg.det(rotations) - 1.0)) @check("valid_rotations") def valid_rotations(episode: Episode, ev: Evidence) -> str | None: - """Fails an episode with an observed pose rotation that is not orthonormal or a root quaternion that is not unit. + """Fails an episode with an observed pose rotation that is not a proper rotation or a root quaternion that is not unit. Both tests hold for either quaternion order, so the check needs no convention. """ @@ -101,16 +103,20 @@ def pose_action_bounds(episode: Episode, ev: Evidence) -> str | None: @check("one_frame_per_step", group="image") def one_frame_per_step(episode: Episode, ev: Evidence) -> str | None: - """Fails an episode whose camera streams do not hold one frame per action.""" + """Fails an episode that lacks a camera stream the dataset records, or whose streams do not hold one frame per action.""" + streams = _image_streams(episode, ev) + missing = sorted(ev.camera_paths - streams.keys()) + if missing: + return f"{', '.join(missing)} not recorded" num_steps = len(episode["actions"]) - bad = [path for path, dataset in _image_streams(episode) if len(dataset) != num_steps] + bad = [path for path, dataset in streams.items() if len(dataset) != num_steps] return f"{', '.join(bad)} do not hold {num_steps} frames" if bad else None @check("no_black_frames", group="image") def no_black_frames(episode: Episode, ev: Evidence) -> str | None: """Fails an episode with a camera frame whose mean pixel value is below :data:`BLACK_FRAME_MEAN`.""" - for path, dataset in _image_streams(episode): + for path, dataset in _image_streams(episode, ev).items(): for start in range(0, len(dataset), FRAME_CHUNK): means = dataset[start : start + FRAME_CHUNK].mean(axis=(1, 2, 3)) if (means < BLACK_FRAME_MEAN).any(): @@ -121,7 +127,7 @@ def no_black_frames(episode: Episode, ev: Evidence) -> str | None: @check("no_frozen_frames", group="image") def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: """Fails an episode whose camera repeats one frame :data:`FROZEN_FRAMES` times in a row.""" - for path, dataset in _image_streams(episode): + for path, dataset in _image_streams(episode, ev).items(): previous, repeats = None, 1 for start in range(0, len(dataset), FRAME_CHUNK): for offset, frame in enumerate(dataset[start : start + FRAME_CHUNK]): @@ -190,7 +196,7 @@ def final_success_rate(ev: Evidence) -> Value: area="validity", signal="data correctness", scope="dataset", - unit="max |R^T R - I|", + unit="max(|R^T R - I|, |det R - 1|)", better="lower", primary="p95", requires=["obs/datagen_info/target_eef_pose"], @@ -198,11 +204,13 @@ def final_success_rate(ev: Evidence) -> Value: ) def action_rotation_error(ev: Evidence) -> Value: """p50 and p95 deviation of the commanded target rotations from a valid rotation, over every step and end effector.""" - errors = np.concatenate([ - _rotation_errors(episode[f"obs/datagen_info/target_eef_pose/{eef}"]) - for episode in ev.episodes - for eef in episode.children("obs/datagen_info/target_eef_pose") - ]) + errors = [] + for episode in ev.episodes: + for eef in episode.children("obs/datagen_info/target_eef_pose"): + targets = episode[f"obs/datagen_info/target_eef_pose/{eef}"] + # A channel that commands no pose records all-zero targets, which are not rotations. + errors.append(_rotation_errors(targets[np.abs(targets).reshape(len(targets), -1).max(axis=1) > 0])) + errors = np.concatenate(errors) if not len(errors): raise NotApplicable("no end effector commands a pose") p50, p95 = np.percentile(errors, [50, 95]) diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index a252907..b34b35b 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -35,17 +35,18 @@ def test_failing_episodes_are_counted_and_left_out_of_dataset_metrics(tmp_path): good = {"actions": actions, "step_success": np.array([False, False, True])} non_finite = {"actions": np.where(np.eye(3, 7) > 0, np.nan, 0.0), "step_success": np.array([True, True, False])} short = {"actions": actions, "step_success": np.array([False, False])} + empty = {"actions": np.zeros((0, 7)), "step_success": np.zeros(0, dtype=bool)} report = build_report( - _write(tmp_path / "d.hdf5", [good, non_finite, short]), + _write(tmp_path / "d.hdf5", [good, non_finite, short, empty]), metrics=_metrics("correctness_pass_rate", "final_success_rate"), ) - assert report["metrics"]["correctness_pass_rate"]["value"] == pytest.approx(1 / 3) + assert report["metrics"]["correctness_pass_rate"]["value"] == pytest.approx(1 / 4) assert report["metrics"]["final_success_rate"]["value"] == 1.0 and report["metrics"]["final_success_rate"]["n"] == 1 assert {name: entry["failed"] for name, entry in report["check_failures"].items() if entry["failed"]} == { "finite": 1, - "aligned_lengths": 1, + "aligned_lengths": 2, } @@ -55,11 +56,15 @@ def test_rotation_and_bound_checks_flag_their_defects(tmp_path): clipped = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose} beyond_bound = {"actions": np.full((3, 7), 1.5), "obs/datagen_info/eef_pose/arm": pose} skewed = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose * [[1.1], [1.0], [1.0], [1.0]]} + zero = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": np.zeros((3, 4, 4))} + reflected = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose * [[-1.0], [1.0], [1.0], [1.0]]} + episodes = [clipped, beyond_bound, skewed, zero, reflected] + entry["exported_episodes"] = len(episodes) - report = build_report(_write(tmp_path / "d.hdf5", [clipped, beyond_bound, skewed], entry), metrics={}) + report = build_report(_write(tmp_path / "d.hdf5", episodes, entry), metrics={}) failures = {name: sorted(entry["examples"]) for name, entry in report["check_failures"].items() if entry["failed"]} - assert failures == {"pose_action_bounds": ["demo_1"], "valid_rotations": ["demo_2"]} + assert failures == {"pose_action_bounds": ["demo_1"], "valid_rotations": ["demo_2", "demo_3", "demo_4"]} def test_image_checks_flag_black_frozen_and_missing_frames(tmp_path): @@ -76,12 +81,17 @@ def frames(count: int) -> np.ndarray: {"actions": np.zeros((steps, 7)), "obs/camera": camera} for camera in (frames(steps), black, frozen, frames(steps - 1)) ] + episodes.append({"actions": np.zeros((steps, 7))}) report = build_report(_write(tmp_path / "d.hdf5", episodes), metrics=_metrics("image_stream_pass_rate")) failures = {name: sorted(entry["examples"]) for name, entry in report["check_failures"].items() if entry["failed"]} - assert failures == {"no_black_frames": ["demo_1"], "no_frozen_frames": ["demo_2"], "one_frame_per_step": ["demo_3"]} - assert report["metrics"]["image_stream_pass_rate"]["value"] == pytest.approx(1 / 4) + assert failures == { + "no_black_frames": ["demo_1"], + "no_frozen_frames": ["demo_2"], + "one_frame_per_step": ["demo_3", "demo_4"], + } + assert report["metrics"]["image_stream_pass_rate"]["value"] == pytest.approx(1 / 5) def test_final_success_rate_reports_the_source_baseline(tmp_path): diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index 14a47b6..f46a0cb 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -91,7 +91,8 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str]) -> None: - """Report a generation run's metrics with ``validate_dataset.py`` and assert exactly ``expected_ok`` computed a value. + """Report a generation run's metrics with ``validate_dataset.py``; assert every episode passes the checks and + exactly ``expected_ok`` computed a value. A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which this catches. @@ -109,7 +110,11 @@ def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str] report_file, ]) with open(report_file, encoding="utf-8") as report_handle: - rows = json.load(report_handle)["metrics"] + report = json.load(report_handle) + assert ( + report["num_valid_episodes"] == report["num_episodes"] + ), f"{report['num_episodes'] - report['num_valid_episodes']} episodes failed checks: {report['check_failures']}" + rows = report["metrics"] ok = {name for name, row in rows.items() if row["status"] == "ok"} assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" From 9cbd00a0262940bda0e27a3d8dbea212cd2f2877 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 10:32:30 -0500 Subject: [PATCH 19/24] refactor: derive skipped checks from the check groups and trim the summary - Evidence records the group of every check considered; a check with no failures entry did not apply, so skipped checks need no attribute. - time_to_dataset reads the annotation mode and setup time with defaults instead of branching. - A failing check's line names its example episodes without an ellipsis. --- autodata_metrics/evidence.py | 6 ++---- autodata_metrics/report.py | 8 ++------ autodata_metrics/validity.py | 6 +++--- autodata_metrics/viability.py | 15 +++++---------- docs/pages/references/metrics.rst | 8 ++++---- .../step_4_validate_dataset.rst | 1 - 6 files changed, 16 insertions(+), 28 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index d692eb2..9b52517 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -61,9 +61,8 @@ class Evidence: Attributes: all_episodes: The dataset's episodes. episodes: The episodes that passed every episode check; dataset metrics read these. - failures: Per episode check that ran, the failing episodes' names and reasons. - check_groups: Per episode check that ran, its group. - skipped_checks: Per episode check that did not apply to any episode, its group. + failures: Per episode check that applied to some episode, the failing episodes' names and reasons. + check_groups: Per episode check considered, its group; a check absent from ``failures`` did not apply. env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer. result: Generation result, or None for a dataset without a run record or an interrupted run. run: The result's ``run`` block, or empty. @@ -83,7 +82,6 @@ def __init__( self.episodes = list(self.all_episodes) self.failures: dict[str, dict[str, str]] = {} self.check_groups: dict[str, str] = {} - self.skipped_checks: dict[str, str] = {} self.result = result self.run: Mapping[str, Any] = (result or {}).get("run", {}) self.attempts = attempts diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 063c3ce..c0f35c3 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -96,8 +96,8 @@ def check_same_run(evidence: Evidence) -> list[str]: def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: """Run the episode checks and keep only the episodes that pass all of them in ``evidence.episodes``.""" for spec in checks.values(): + evidence.check_groups[spec.name] = spec.group if any(not evidence.has(key) for key in spec.requires): - evidence.skipped_checks[spec.name] = spec.group continue failed: dict[str, str] = {} applied = False @@ -113,9 +113,6 @@ def run_checks(evidence: Evidence, checks: Mapping[str, CheckSpec]) -> None: failed[episode.name] = reason if applied: evidence.failures[spec.name] = failed - evidence.check_groups[spec.name] = spec.group - else: - evidence.skipped_checks[spec.name] = spec.group failing = {name for failed in evidence.failures.values() for name in failed} evidence.episodes = [episode for episode in evidence.all_episodes if episode.name not in failing] @@ -302,9 +299,8 @@ def format_report(report: Mapping[str, Any], gate: Mapping[str, str | None] | No lines = [" · ".join(title), " · ".join(verdict)] for name, entry in report["check_failures"].items(): if entry["failed"]: - more = ", …" if entry["failed"] > len(entry["examples"]) else "" lines.append( - f" ✗ {name} failed {_count(entry['failed'], 'episodes')}: {', '.join(entry['examples'])}{more}" + f" ✗ {name} failed {_count(entry['failed'], 'episodes')}, e.g. {', '.join(entry['examples'])}" ) rows = [] diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 3397210..8d3c0c0 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -142,14 +142,14 @@ def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: def _pass_rate(ev: Evidence, group: str, none_apply: str) -> Value: - names = [name for name, check_group in ev.check_groups.items() if check_group == group] + considered = [name for name, check_group in ev.check_groups.items() if check_group == group] + names = [name for name in considered if name in ev.failures] if not names: raise NotApplicable(none_apply) if not ev.all_episodes: raise NotApplicable("the dataset has no episodes") failing = {episode for name in names for episode in ev.failures[name]} - skipped = [name for name, check_group in ev.skipped_checks.items() if check_group == group] - details = {"checks_run": names, "checks_skipped": skipped} + details = {"checks_run": names, "checks_skipped": [name for name in considered if name not in ev.failures]} return Value(1 - len(failing) / len(ev.all_episodes), n=len(ev.all_episodes), details=details) diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index d5a2bbc..1f6fad9 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -106,17 +106,12 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: def time_to_dataset(ev: Evidence) -> Value: """Wall time of the automated steps, each process timed from its start: annotating the source and generating.""" annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} - automated = {"setup_s": ev.run["setup_time_s"], "generation_s": ev.run["generation_time_s"]} - manual = {} + manual = annotation.get("mode") == "manual" + parts = {"setup_s": ev.run["setup_time_s"], "generation_s": ev.run["generation_time_s"]} if annotation: - # An entry written before the mode was recorded counts as automatic, as in version 1. - if annotation.get("mode", "auto") == "auto": - automated["annotation_s"] = annotation.get("setup_time_s", 0.0) + annotation["time_s"] - else: - automated["annotation_s"] = annotation["setup_time_s"] - manual["manual_annotation_s"] = annotation["time_s"] - details = {**automated, **manual, "note": "manual recording and manual annotation are not in the total"} - return Value(sum(automated.values()), n=1, details=details) + parts["annotation_s"] = annotation.get("setup_time_s", 0.0) + (0.0 if manual else annotation["time_s"]) + details = {**parts, "manual_annotation_s": annotation["time_s"] if manual else None} + return Value(sum(parts.values()), n=1, details=details) @metric( diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index d5ed482..ba6390b 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -35,13 +35,13 @@ The command prints a summary; ``--report_file`` writes the full report, with eve not reported by this algorithm: planning_failure_rate, planning_time needs a --source recorded with --record_metric_inputs: annotation_pass_rate -- The second line is the verdict: valid episodes, how many metrics were computed, and the gate. +- The second line is the verdict: valid episodes, how many metrics were computed, and, with ``--gate``, + the gate, whose metrics are marked ✓ or ✗. - **Value** is the metric's value; a metric with percentiles shows the one a gate checks by default. **Source** is the same metric on the ``--source`` dataset. **Based on** is the sample count, with a rate's 95% interval and, when some checks did not apply, how many checks a pass rate rests on. -- With ``--gate``, ✓ and ✗ mark the gated metrics. -- **Not computed** groups the metrics that do not apply by what would make them apply. -- A metric that fails is listed under **Errors**; the other metrics are still computed. +- **Not computed** groups the metrics that do not apply by what would make them apply. A metric that + fails is listed under **Errors**; the other metrics are still computed. The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own diff --git a/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst b/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst index 79f50b2..7fddc01 100644 --- a/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst +++ b/docs/pages/workflows/franka_cube_stack_mimicgen/step_4_validate_dataset.rst @@ -29,7 +29,6 @@ For a dataset generated with ``--record_metric_inputs`` and ``--result_file``, r --result_file ./datasets/generated_dataset_mimicgen_franka.json \ --source ./datasets/annotated_datasets/dataset_franka_annotated.hdf5 -The summary lists each metric's value, the metrics that do not apply and what would make them apply. See :doc:`../../references/metrics` for what each metric means and how to gate on it. From b9c9d5b4f799caabea8f9252116af012a957bf8c Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 11:11:02 -0500 Subject: [PATCH 20/24] fix: report degenerate data instead of failing on it - aligned_lengths skips scalar datasets, and valid_rotations reads the last axis only of arrays that have one. - action_rotation_error is not applicable to a target group without end effectors. - The console summary leaves out an interval with a missing bound, shows every entry of a dict value without its primary entry, and a sample count that is not an integer makes its metric an error. - time_to_dataset counts annotation time only when the record says it was automatic; other annotation time is reported apart. --- autodata_metrics/report.py | 13 ++++++---- autodata_metrics/validity.py | 9 ++++--- autodata_metrics/viability.py | 13 +++++++--- autodata_tests/metrics/test_report.py | 31 +++++++++++++++++++++++- autodata_tests/metrics/test_viability.py | 6 +++-- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index c0f35c3..f1e6e3e 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -142,6 +142,8 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} if not _is_finite(value.value): return {**row, "status": "error", "reason": f"non-finite value {value.value}"} + if not isinstance(value.n, numbers.Integral): + return {**row, "status": "error", "reason": f"sample count {value.n!r} is not an integer"} row = {**row, "status": "ok", "value": value.value, "n": value.n, "details": _finite_or_none(value.details)} if spec.baseline and evidence.source is not None: row["baseline"] = evaluate(spec, evidence.source).get("value") @@ -349,12 +351,13 @@ def _count(count: int, noun: str) -> str: def _based_on(row: Mapping[str, Any]) -> str: """The sample count, with a rate's 95% interval and a pass rate's check coverage.""" parts = [_count(row["n"], row["sample"])] - interval = row["details"].get("interval_95") - if interval: + interval = row["details"].get("interval_95") or [] + if len(interval) == 2 and all(isinstance(bound, int | float) for bound in interval): low, high = (_console_number(bound, row["unit"]) for bound in interval) parts.append(f"95% CI {low.removesuffix('%')}–{high}") - if row["details"].get("checks_skipped"): - run, skipped = len(row["details"]["checks_run"]), len(row["details"]["checks_skipped"]) + skipped = len(row["details"].get("checks_skipped") or []) + if skipped: + run = len(row["details"].get("checks_run") or []) parts.append(f"{run} of {run + skipped} checks") return " · ".join(parts) @@ -363,7 +366,7 @@ def _console_value(value: Any, row: Mapping[str, Any], labelled: bool = True) -> """A value, or a dict value's primary entry (every entry without one), as the console shows it.""" if not isinstance(value, Mapping): return _console_number(value, row["unit"]) - keys = [row["primary"]] if row["primary"] else list(value) + keys = [row["primary"]] if row["primary"] in value else list(value) return ", ".join((f"{key} " if labelled else "") + _console_number(value[key], row["unit"]) for key in keys) diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 8d3c0c0..c288ea1 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -54,7 +54,10 @@ def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: bad = [ path for path, dataset in episode.datasets() - if not path.startswith(PER_EPISODE_PATHS) and path not in ev.camera_paths and len(dataset) != num_steps + if dataset.ndim > 0 + and not path.startswith(PER_EPISODE_PATHS) + and path not in ev.camera_paths + and len(dataset) != num_steps ] return f"{', '.join(bad)} do not hold {num_steps} steps" if bad else None @@ -77,7 +80,7 @@ def valid_rotations(episode: Episode, ev: Evidence) -> str | None: for path, dataset in episode.datasets(): if path.startswith(OBSERVED_POSE_PATHS) and dataset.shape[-2:] == (4, 4): error = _rotation_errors(dataset[()]).max(initial=0.0) - elif path.endswith("root_pose") and dataset.shape[-1] == 7: + elif path.endswith("root_pose") and dataset.shape[-1:] == (7,): error = np.abs(np.linalg.norm(dataset[()][..., 3:7], axis=-1) - 1.0).max(initial=0.0) else: continue @@ -218,7 +221,7 @@ def action_rotation_error(ev: Evidence) -> Value: targets = episode[f"obs/datagen_info/target_eef_pose/{eef}"] # A channel that commands no pose records all-zero targets, which are not rotations. errors.append(_rotation_errors(targets[np.abs(targets).reshape(len(targets), -1).max(axis=1) > 0])) - errors = np.concatenate(errors) + errors = np.concatenate(errors) if errors else np.empty(0) if not len(errors): raise NotApplicable("no end effector commands a pose") p50, p95 = np.percentile(errors, [50, 95]) diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 1f6fad9..7f94094 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -104,13 +104,18 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: version=2, ) def time_to_dataset(ev: Evidence) -> Value: - """Wall time of the automated steps, each process timed from its start: annotating the source and generating.""" + """Wall time of the automated steps, each process timed from its start: annotating the source and generating. + + Annotation that was manual, or recorded without its mode, is reported in ``uncounted_annotation_s`` instead. + """ annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} - manual = annotation.get("mode") == "manual" parts = {"setup_s": ev.run["setup_time_s"], "generation_s": ev.run["generation_time_s"]} + uncounted = None if annotation: - parts["annotation_s"] = annotation.get("setup_time_s", 0.0) + (0.0 if manual else annotation["time_s"]) - details = {**parts, "manual_annotation_s": annotation["time_s"] if manual else None} + automatic = annotation.get("mode") == "auto" + parts["annotation_s"] = annotation.get("setup_time_s", 0.0) + (annotation["time_s"] if automatic else 0.0) + uncounted = None if automatic else annotation["time_s"] + details = {**parts, "annotation_mode": annotation.get("mode"), "uncounted_annotation_s": uncounted} return Value(sum(parts.values()), n=1, details=details) diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index d6fd750..12ecb55 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -5,14 +5,16 @@ import h5py import json +import numpy as np import re import subprocess import sys +from dataclasses import replace from pathlib import Path import pytest -from autodata_metrics import MetricSpec, Value, build_report, format_report, gate_results +from autodata_metrics import REGISTERED_METRICS, MetricSpec, Value, build_report, format_report, gate_results def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: @@ -160,6 +162,33 @@ def test_console_summary_marks_the_gate_and_says_how_to_compute_the_rest(tmp_pat assert "not reported by this algorithm: planned" in summary +def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp_path): + path = tmp_path / "d.hdf5" + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) + odd = data.create_group("demo_0") + odd.create_dataset("actions", data=np.zeros((2, 7))) + odd.create_dataset("obs/robot/root_pose", data=1.0) + odd.create_group("obs/datagen_info/target_eef_pose") + data.create_group("demo_1").create_dataset("actions", data=np.zeros((0, 7))) + metrics = { + "action_rotation_error": REGISTERED_METRICS["action_rotation_error"], + "open": _spec("open", lambda ev: Value(0.5, n=2, details={"interval_95": [0.1, float("inf")]})), + "partial": replace(_spec("partial", lambda ev: Value({"p50": 1.0}, n=1)), primary="p95"), + "uncounted": _spec("uncounted", lambda ev: Value(1.0, n=None)), + } + + report = build_report(str(path), metrics=metrics) + + summary = format_report(report) + json.dumps(report, allow_nan=False) + assert report["num_valid_episodes"] == 1 and "aligned_lengths" in report["check_failures"] + assert report["metrics"]["action_rotation_error"]["status"] == "not_applicable" + assert re.search(r"open +50% +2 samples$", summary, re.M) and re.search(r"partial +p50 100%", summary) + assert report["metrics"]["uncounted"]["status"] == "error" + + def test_metrics_import_without_torch_or_isaaclab(): code = "import sys, autodata_metrics; print(sorted({'torch', 'isaaclab', 'autodata_core'} & set(sys.modules)))" output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout diff --git a/autodata_tests/metrics/test_viability.py b/autodata_tests/metrics/test_viability.py index f8fe5ac..c0afed0 100644 --- a/autodata_tests/metrics/test_viability.py +++ b/autodata_tests/metrics/test_viability.py @@ -30,7 +30,7 @@ def test_attempts_before_stop_keeps_attempts_started_before_the_first_env_finish assert attempts_before_stop(short_failures, num_envs=2) is None -def test_time_to_dataset_counts_automated_steps_and_keeps_manual_annotation_apart(): +def test_time_to_dataset_counts_only_annotation_recorded_as_automatic(): run = {"setup_time_s": 20.0, "generation_time_s": 80.0} def measured(annotation: dict): @@ -40,6 +40,8 @@ def measured(annotation: dict): automatic = measured({"mode": "auto", "setup_time_s": 15.0, "time_s": 60.0}) manual = measured({"mode": "manual", "setup_time_s": 15.0, "time_s": 600.0}) + unknown = measured({"time_s": 60.0}) assert automatic.value == 175.0 - assert (manual.value, manual.details["manual_annotation_s"]) == (115.0, 600.0) + assert (manual.value, manual.details["uncounted_annotation_s"]) == (115.0, 600.0) + assert (unknown.value, unknown.details["uncounted_annotation_s"]) == (100.0, 60.0) From 622af85b5fed3f0700db762e2f6025ef3e314988 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 11:34:07 -0500 Subject: [PATCH 21/24] fix: keep the report and its summary whole on malformed metric output and gates - A metric that returns something other than a Value, a nested dict, details that are not a mapping, or a non-integer sample count is an error row instead of breaking the report or its summary. - Details and value keys are written as strings, so the report stays valid JSON. - env_args that are not JSON read as empty. - load_gate validates the gate file before the report is built and reads numbers YAML leaves as strings, such as 1e-3; a key on a value without entries fails the gate instead of raising. - Not-computed hints name the absent attempt log or source before the algorithm, and a result file that does not exist is warned about. - The report records the source dataset's checks, and the summary's title says how many source episodes were valid. - A rate just below one shows as >99.9% rather than 100%. --- autodata_metrics/__init__.py | 2 +- autodata_metrics/evidence.py | 5 +- autodata_metrics/report.py | 171 ++++++++++++++++-------- autodata_metrics/validity.py | 4 +- autodata_tests/metrics/test_report.py | 58 +++++--- autodata_tests/metrics/test_validity.py | 9 +- docs/pages/references/metrics.rst | 5 +- scripts/validate_dataset.py | 14 +- 8 files changed, 180 insertions(+), 88 deletions(-) diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py index f75f7e1..8897644 100644 --- a/autodata_metrics/__init__.py +++ b/autodata_metrics/__init__.py @@ -20,4 +20,4 @@ check, metric, ) -from autodata_metrics.report import build_report, format_report, gate_results +from autodata_metrics.report import build_report, format_report, gate_results, load_gate diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 9b52517..60a9456 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -25,7 +25,10 @@ def _has_path(record: Any, dotted_key: str) -> bool: def read_env_args(h5_file: h5py.File) -> dict[str, Any]: """The dataset's ``env_args``, or empty when they are not a JSON object.""" raw_env_args = h5_file["data"].attrs.get("env_args", "{}") - env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + try: + env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + except ValueError: + return {} return env_args if isinstance(env_args, dict) else {} diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index f1e6e3e..7647f16 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -18,7 +18,15 @@ from typing import Any from autodata_metrics.evidence import Evidence, read_env_args -from autodata_metrics.registry import AREAS, REGISTERED_CHECKS, REGISTERED_METRICS, CheckSpec, MetricSpec, NotApplicable +from autodata_metrics.registry import ( + AREAS, + REGISTERED_CHECKS, + REGISTERED_METRICS, + CheckSpec, + MetricSpec, + NotApplicable, + Value, +) from autodata_utils.generation_result import attempts_file_for, code_revision, input_record SCHEMA_VERSION = 1 @@ -34,7 +42,7 @@ def read_run( """ result_path = Path(result_file) result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None - warnings = [] + warnings = [] if result_path.exists() else [f"{result_file} does not exist"] if result is not None and run_id is not None and result.get("run", {}).get("run_id") != run_id: warnings.append(f"{result_file} belongs to another run, so the run is reported from its attempt log") result = None @@ -133,6 +141,8 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": "no valid episodes"} missing = next((key for key in spec.requires if not evidence.has(key)), None) if missing is not None: + head = missing.partition(".")[0] + missing = head if head in ("attempts", "source") and not evidence.has(head) else missing return {**row, "status": "not_applicable", "reason": f"missing {missing}", "missing": missing} try: value = spec.compute(evidence) @@ -140,28 +150,45 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": str(exc)} except Exception as exc: # A failing metric must not sink the report of every other metric. return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} - if not _is_finite(value.value): - return {**row, "status": "error", "reason": f"non-finite value {value.value}"} - if not isinstance(value.n, numbers.Integral): - return {**row, "status": "error", "reason": f"sample count {value.n!r} is not an integer"} - row = {**row, "status": "ok", "value": value.value, "n": value.n, "details": _finite_or_none(value.details)} + problem = _unreportable(value) + if problem is not None: + return {**row, "status": "error", "reason": problem} + row = { + **row, + "status": "ok", + "value": _finite_or_none(value.value), + "n": int(value.n), + "details": _finite_or_none(value.details), + } if spec.baseline and evidence.source is not None: row["baseline"] = evaluate(spec, evidence.source).get("value") return row +def _unreportable(value: Any) -> str | None: + """Why a metric's return value cannot be reported, or None.""" + if not isinstance(value, Value): + return f"returned {type(value).__name__}, not a Value" + if not _is_finite(value.value): + return f"value {value.value!r} is not a finite number or a mapping of finite numbers" + if not isinstance(value.n, numbers.Integral): + return f"sample count {value.n!r} is not an integer" + if not isinstance(value.details, Mapping): + return f"details {value.details!r} are not a mapping" + return None + + def _is_finite(value: Any) -> bool: """Whether ``value`` is a finite number, or a mapping of finite numbers.""" - if isinstance(value, Mapping): - return all(_is_finite(item) for item in value.values()) - return isinstance(value, numbers.Real) and math.isfinite(value) + items = value.values() if isinstance(value, Mapping) else [value] + return all(isinstance(item, numbers.Real) and math.isfinite(item) for item in items) def _finite_or_none(value: Any) -> Any: """``value`` with every non-finite number replaced by None, so the report stays valid JSON.""" value = value.tolist() if hasattr(value, "tolist") else value if isinstance(value, Mapping): - return {key: _finite_or_none(item) for key, item in value.items()} + return {str(key): _finite_or_none(item) for key, item in value.items()} if isinstance(value, list | tuple): return [_finite_or_none(item) for item in value] return None if isinstance(value, float) and not math.isfinite(value) else value @@ -202,12 +229,8 @@ def build_report( run_checks(checked, REGISTERED_CHECKS if checks is None else checks) metrics = REGISTERED_METRICS if metrics is None else metrics rows = {name: evaluate(spec, evidence) for name, spec in metrics.items()} - num_episodes = len(evidence.all_episodes) - num_valid = len(evidence.episodes) - failures = { - name: {"failed": len(failed), "examples": dict(list(failed.items())[:3])} - for name, failed in evidence.failures.items() - } + checked = _check_summary(evidence) + source_checked = _check_summary(source_evidence) if source_evidence is not None else None return { "schema_version": SCHEMA_VERSION, "inputs": { @@ -217,45 +240,79 @@ def build_report( "run_id": run_id, "revision": code_revision(), }, - "num_episodes": num_episodes, - "num_valid_episodes": num_valid, - "check_failures": failures, + **checked, + "source_checks": source_checked, "warnings": warnings, "metrics": rows, } -def gate_results(report: Mapping[str, Any], gate_file: str) -> dict[str, str | None]: - """Per metric in ``gate_file``, None when the report meets its bounds, else why it does not. +def _check_summary(evidence: Evidence) -> dict[str, Any]: + return { + "num_episodes": len(evidence.all_episodes), + "num_valid_episodes": len(evidence.episodes), + "check_failures": { + name: {"failed": len(failed), "examples": dict(list(failed.items())[:3])} + for name, failed in evidence.failures.items() + }, + } + - The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the - metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or - at another version fails. +def load_gate(gate_file: str) -> dict[str, dict[str, Any]]: + """The gate in ``gate_file``: per metric, optional ``min``/``max`` bounds, a ``key`` into a dict value (default: the + metric's primary entry) and a ``version``. Raises: - ValueError: If a gate entry has a bound that is not a finite number, or bounds a dict value without a - ``key`` and the metric has no primary entry. + ValueError: If the file does not map metric names to bounds, or a bound is not a finite number. + """ + gate = yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")) + if not isinstance(gate, Mapping): + raise ValueError(f"{gate_file} must map metric names to bounds, such as 'name: {{min: 0.5}}'.") + entries = {} + for name, bound in gate.items(): + bound = {} if bound is None else bound + if not isinstance(bound, Mapping): + raise ValueError(f"Gate entry {name!r} must map bounds, such as {{min: 0.5}}, not {bound!r}.") + entries[name] = {**bound, **{side: _bound(name, bound[side]) for side in ("min", "max") if side in bound}} + return entries + + +def _bound(name: str, limit: Any) -> float: + """A gate bound as a number: YAML reads ``1e-3`` as a string, and ``true`` as a bool, which is not a bound.""" + number = limit + if isinstance(limit, str): + with contextlib.suppress(ValueError): + number = float(limit) + if not isinstance(number, int | float) or isinstance(number, bool) or not math.isfinite(number): + raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limit!r}.") + return number + + +def gate_results(report: Mapping[str, Any], gate: Mapping[str, Mapping[str, Any]]) -> dict[str, str | None]: + """Per gated metric, None when the report meets its bounds, else why it does not. + + A gated metric that is missing, not applicable, errored or at another version fails. + + Args: + report: A report from :func:`build_report`. + gate: A gate from :func:`load_gate`. """ - results = {} - for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): - limits = [bound[side] for side in ("min", "max") if side in bound] - if not all(_is_bound(limit) for limit in limits): - raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") - results[name] = _gate_failure(name, report["metrics"].get(name), bound) - return results + return {name: _gate_failure(report["metrics"].get(name), bound) for name, bound in gate.items()} -def _gate_failure(name: str, row: Mapping[str, Any] | None, bound: Mapping[str, Any]) -> str | None: +def _gate_failure(row: Mapping[str, Any] | None, bound: Mapping[str, Any]) -> str | None: if row is None or row["status"] != "ok": return "not reported" if row is None else row["status"] if "version" in bound and bound["version"] != row["version"]: return f"version {row['version']}, gate written for version {bound['version']}" - key = bound.get("key", row["primary"]) - value = row["value"].get(key) if key else row["value"] + value = row["value"] if isinstance(value, Mapping): - raise ValueError(f"Gate entry {name!r} needs a key into {sorted(value)}.") - if not _is_finite(value): - return f"{value!r} is not a finite number" + key = bound.get("key", row["primary"]) + if key not in value: + return f"needs a key into {sorted(value)}" + value = value[key] + elif "key" in bound: + return f"key {bound['key']!r} given for a value without entries" if "min" in bound and value < bound["min"]: return f"{value:.4g} is below {bound['min']}" if "max" in bound and value > bound["max"]: @@ -263,11 +320,6 @@ def _gate_failure(name: str, row: Mapping[str, Any] | None, bound: Mapping[str, return None -def _is_bound(limit: Any) -> bool: - """Whether a gate bound is a finite number; YAML reads ``true`` and ``false`` as bools, which are not bounds.""" - return isinstance(limit, int | float) and not isinstance(limit, bool) and math.isfinite(limit) - - CONSOLE_WIDTH = 100 """Width the console summary wraps its lists of metrics to.""" @@ -289,7 +341,11 @@ def format_report(report: Mapping[str, Any], gate: Mapping[str, str | None] | No title = [f"AutoData metrics · {Path(inputs['dataset']['path']).name}"] title += [f"run {inputs['run_id'][:8]}"] if inputs["run_id"] else [] - title += [f"source {Path(inputs['source']['path']).name}"] if inputs["source"] else [] + if inputs["source"]: + valid, total = report["source_checks"]["num_valid_episodes"], report["source_checks"]["num_episodes"] + title.append( + f"source {Path(inputs['source']['path']).name}" + (f" ({valid}/{total} valid)" if valid < total else "") + ) verdict = [f"{report['num_valid_episodes']}/{report['num_episodes']} episodes valid", f"{len(computed)} computed"] verdict += [f"{len(skipped)} not computed"] if skipped else [] verdict += [_count(len(errors), "errors")] if errors else [] @@ -351,14 +407,12 @@ def _count(count: int, noun: str) -> str: def _based_on(row: Mapping[str, Any]) -> str: """The sample count, with a rate's 95% interval and a pass rate's check coverage.""" parts = [_count(row["n"], row["sample"])] - interval = row["details"].get("interval_95") or [] - if len(interval) == 2 and all(isinstance(bound, int | float) for bound in interval): + interval, run, skipped = (row["details"].get(key) for key in ("interval_95", "checks_run", "checks_skipped")) + if isinstance(interval, list) and len(interval) == 2 and all(isinstance(bound, int | float) for bound in interval): low, high = (_console_number(bound, row["unit"]) for bound in interval) parts.append(f"95% CI {low.removesuffix('%')}–{high}") - skipped = len(row["details"].get("checks_skipped") or []) - if skipped: - run = len(row["details"].get("checks_run") or []) - parts.append(f"{run} of {run + skipped} checks") + if isinstance(run, list) and isinstance(skipped, list) and skipped: + parts.append(f"{len(run)} of {len(run) + len(skipped)} checks") return " · ".join(parts) @@ -374,7 +428,8 @@ def _console_number(value: float, unit: str) -> str: """Fractions as percentages, seconds as durations; other units follow the number unless they hold a space, which marks a description of the quantity rather than a unit.""" if unit.startswith("fraction"): - return f"{100 * value:.3g}%" + percent = f"{100 * value:.3g}%" + return ">99.9%" if percent == "100%" and value < 1 else percent if unit == "s" or unit.startswith("s "): return _duration(value) text = f"{value:,.0f}" if abs(value) >= 100 else re.sub(r"e\+?(-?)0*(\d)", r"e\1\2", f"{value:.3g}") @@ -399,12 +454,14 @@ def _not_computed_because(row: Mapping[str, Any], inputs: Mapping[str, Any]) -> missing = row.get("missing") if missing is None: return row["reason"] - if missing.startswith("attempts.extras."): - return "not reported by this algorithm" - if missing.partition(".")[0] == "source": - return "needs --source" if inputs["source"] is None else "needs a --source recorded with --record_metric_inputs" + if missing == "source": + return "needs --source" + if missing.startswith("source."): + return "needs a --source recorded with --record_metric_inputs" if missing == "attempts" or missing.startswith("run."): return "needs --result_file" if inputs["result_file"] is None else "needs --record_metric_inputs" + if missing.startswith("attempts.extras."): + return "not reported by this algorithm" if missing.startswith(_METRIC_INPUT_KEYS): return "needs --record_metric_inputs" return row["reason"] diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index c288ea1..04df46a 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -145,12 +145,12 @@ def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: def _pass_rate(ev: Evidence, group: str, none_apply: str) -> Value: + if not ev.all_episodes: + raise NotApplicable("the dataset has no episodes") considered = [name for name, check_group in ev.check_groups.items() if check_group == group] names = [name for name in considered if name in ev.failures] if not names: raise NotApplicable(none_apply) - if not ev.all_episodes: - raise NotApplicable("the dataset has no episodes") failing = {episode for name in names for episode in ev.failures[name]} details = {"checks_run": names, "checks_skipped": [name for name in considered if name not in ev.failures]} return Value(1 - len(failing) / len(ev.all_episodes), n=len(ev.all_episodes), details=details) diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 12ecb55..c8b57e0 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -14,7 +14,7 @@ import pytest -from autodata_metrics import REGISTERED_METRICS, MetricSpec, Value, build_report, format_report, gate_results +from autodata_metrics import REGISTERED_METRICS, MetricSpec, Value, build_report, format_report, gate_results, load_gate def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: @@ -111,23 +111,38 @@ def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): build_report(dataset, result_file=result_file, source=_dataset(tmp_path / "source.hdf5"), metrics={}) -def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): +def test_gate_flags_violations_and_refuses_a_malformed_gate(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} missing = {"status": "ok", "value": {"p50": 1.0}, "primary": "p95", "version": 1} - report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}, "gap": missing}} + metrics = { + "low": row, + "old": row, + "fine": row, + "keyed": row, + "absent": {"status": "not_applicable"}, + "gap": missing, + } gate = tmp_path / "gate.yaml" gate.write_text( - "low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\ngap: {max: 2}\n" + "low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 5e-1, version: 2}\nkeyed: {key: p95}\n" + "absent: {min: 0}\ngap: {max: 2}\n" ) - results = gate_results(report, str(gate)) + results = gate_results({"metrics": metrics}, load_gate(str(gate))) - assert [name for name, reason in results.items() if reason is not None] == ["low", "old", "absent", "gap"] + assert [name for name, reason in results.items() if reason is not None] == ["low", "old", "keyed", "absent", "gap"] assert results["fine"] is None - for bad_bound in (".nan", "false", "{p95: 1}"): - gate.write_text(f"absent: {{min: {bad_bound}}}\n") - with pytest.raises(ValueError, match="not a finite number"): - gate_results(report, str(gate)) + for malformed in ( + "absent: {min: .nan}", + "absent: {min: false}", + "absent: {min: {p95: 1}}", + "absent: 0.5", + "[low]", + "", + ): + gate.write_text(malformed + "\n") + with pytest.raises(ValueError): + load_gate(str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): @@ -140,7 +155,7 @@ def test_non_finite_numbers_never_reach_the_report(tmp_path): gate.write_text("m: {min: 0}\n") assert report["metrics"]["m"]["status"] == "error" - assert gate_results(report, str(gate)) == {"m": "error"} + assert gate_results(report, load_gate(str(gate))) == {"m": "error"} assert report["metrics"]["d"]["details"] == {"interval_95": [0.5, None]} @@ -154,19 +169,19 @@ def test_console_summary_marks_the_gate_and_says_how_to_compute_the_rest(tmp_pat gate = tmp_path / "gate.yaml" gate.write_text("rate: {min: 0.5}\n") - summary = format_report(report, gate_results(report, str(gate))) + summary = format_report(report, gate_results(report, load_gate(str(gate)))) assert "gate failed on 1 of 1" in summary assert re.search(r"✗ rate +25% +4 samples", summary) assert "needs --record_metric_inputs: held" in summary - assert "not reported by this algorithm: planned" in summary + assert "needs --result_file: planned" in summary def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp_path): path = tmp_path / "d.hdf5" with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) + data.attrs["env_args"] = "{'env_name': 'not JSON'}" odd = data.create_group("demo_0") odd.create_dataset("actions", data=np.zeros((2, 7))) odd.create_dataset("obs/robot/root_pose", data=1.0) @@ -176,17 +191,28 @@ def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp "action_rotation_error": REGISTERED_METRICS["action_rotation_error"], "open": _spec("open", lambda ev: Value(0.5, n=2, details={"interval_95": [0.1, float("inf")]})), "partial": replace(_spec("partial", lambda ev: Value({"p50": 1.0}, n=1)), primary="p95"), + "almost": _spec("almost", lambda ev: Value(0.9995, n=2000, details={np.int64(3): 1})), "uncounted": _spec("uncounted", lambda ev: Value(1.0, n=None)), + "bare": _spec("bare", lambda ev: 0.5), + "nested": _spec("nested", lambda ev: Value({"left": {"p95": 0.1}}, n=1)), + "loose": _spec("loose", lambda ev: Value(0.5, n=1, details=None)), } - report = build_report(str(path), metrics=metrics) + report = build_report(str(path), result_file=str(tmp_path / "typo.json"), metrics=metrics) summary = format_report(report) json.dumps(report, allow_nan=False) assert report["num_valid_episodes"] == 1 and "aligned_lengths" in report["check_failures"] assert report["metrics"]["action_rotation_error"]["status"] == "not_applicable" assert re.search(r"open +50% +2 samples$", summary, re.M) and re.search(r"partial +p50 100%", summary) - assert report["metrics"]["uncounted"]["status"] == "error" + assert re.search(r"almost +>99.9%", summary) + assert {name for name, row in report["metrics"].items() if row["status"] == "error"} == { + "uncounted", + "bare", + "nested", + "loose", + } + assert "typo.json does not exist" in report["warnings"][0] def test_metrics_import_without_torch_or_isaaclab(): diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index 4d70501..4af7b45 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -100,13 +100,14 @@ def frames(count: int) -> np.ndarray: def test_final_success_rate_reports_the_source_baseline(tmp_path): ending = [{"actions": np.zeros((2, 7)), "step_success": np.array([True, held])} for held in (True, False)] - source = _write(tmp_path / "source.hdf5", ending[:1]) + broken = {"actions": np.full((2, 7), np.nan), "step_success": np.array([True, False])} + source = _write(tmp_path / "source.hdf5", [ending[0], broken]) - row = build_report(_write(tmp_path / "d.hdf5", ending), source=source, metrics=_metrics("final_success_rate"))[ - "metrics" - ]["final_success_rate"] + report = build_report(_write(tmp_path / "d.hdf5", ending), source=source, metrics=_metrics("final_success_rate")) + row = report["metrics"]["final_success_rate"] assert (row["value"], row["baseline"]) == (0.5, 1.0) + assert (report["source_checks"]["num_valid_episodes"], report["source_checks"]["num_episodes"]) == (1, 2) def test_action_rotation_error_measures_commanded_rotations(tmp_path): diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index ba6390b..26c1f5d 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -38,8 +38,9 @@ The command prints a summary; ``--report_file`` writes the full report, with eve - The second line is the verdict: valid episodes, how many metrics were computed, and, with ``--gate``, the gate, whose metrics are marked ✓ or ✗. - **Value** is the metric's value; a metric with percentiles shows the one a gate checks by default. - **Source** is the same metric on the ``--source`` dataset. **Based on** is the sample count, with a - rate's 95% interval and, when some checks did not apply, how many checks a pass rate rests on. + **Source** is the same metric on the ``--source`` dataset's valid episodes. **Based on** is the sample + count, with a rate's 95% interval and, when some checks did not apply, how many checks a pass rate rests + on. - **Not computed** groups the metrics that do not apply by what would make them apply. A metric that fails is listed under **Errors**; the other metrics are still computed. diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index 440da73..a2b7a1a 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -23,7 +23,7 @@ import os import sys -from autodata_metrics import build_report, format_report, gate_results +from autodata_metrics import build_report, format_report, gate_results, load_gate REQUIRED_DEMO_FIELDS = { "actions": h5py.Dataset, @@ -181,6 +181,10 @@ def main(argv: list[str] | None = None) -> int: given = [f"--{name}" for name in ("result_file", "source", "metrics", "gate", "report_file") if getattr(args, name)] if given and not args.report: parser.error(f"{', '.join(given)} only apply with --report") + try: + gate = load_gate(args.gate) if args.gate else None + except ValueError as exc: + parser.error(str(exc)) results = [validate_file(path) for path in args.files] print_results(results) @@ -196,15 +200,15 @@ def main(argv: list[str] | None = None) -> int: for module in args.metrics: importlib.import_module(module) report = build_report(args.files[0], result_file=args.result_file, source=args.source) - gate = gate_results(report, args.gate) if args.gate else None + results = gate_results(report, gate) if gate is not None else None print() - print(format_report(report, gate)) + print(format_report(report, results)) if args.report_file: with open(args.report_file, "w", encoding="utf-8") as report_handle: json.dump(report, report_handle, indent=2, default=_json_value) report_handle.write("\n") - if gate is not None: - violations = [f"{name}: {reason}" for name, reason in gate.items() if reason is not None] + if results is not None: + violations = [f"{name}: {reason}" for name, reason in results.items() if reason is not None] print("\nGate passed." if not violations else "\nGate failed:\n " + "\n ".join(violations)) return 1 if num_invalid or violations else 0 From 1b74d5b1cd67248ee65a056da35f611950f9b787 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 11:59:47 -0500 Subject: [PATCH 22/24] fix: refuse corrupt run metadata and malformed gate files - A dataset or source whose env_args are not a JSON object, or whose autodata entry lacks its run id or exported episode count, is refused instead of read as having none, so a run is never matched to it on episode count alone. - validate_dataset.py prints a refused report as one error line, and refuses a --source the validator cannot read. - load_gate refuses unreadable YAML, repeated keys, unknown fields, a key that is not an entry name and a version that is not an integer. - A result file or attempt log that does not parse is named in the error. - The not-computed hint asks for a --source annotated with --record_metric_inputs. --- autodata_metrics/evidence.py | 22 +++++++-- autodata_metrics/report.py | 48 +++++++++++++++---- autodata_tests/metrics/test_report.py | 13 ++++- autodata_tests/utils/test_validate_dataset.py | 11 +++++ docs/pages/references/metrics.rst | 5 +- scripts/validate_dataset.py | 10 +++- 6 files changed, 93 insertions(+), 16 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 60a9456..5df3727 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -23,13 +23,27 @@ def _has_path(record: Any, dotted_key: str) -> bool: def read_env_args(h5_file: h5py.File) -> dict[str, Any]: - """The dataset's ``env_args``, or empty when they are not a JSON object.""" + """The dataset's ``env_args``, or empty when it records none. + + Raises: + ValueError: If ``env_args`` is not a JSON object, or its ``autodata`` entry lacks the run id and exported + episode count; a run cannot be verified against corrupt metadata. + """ raw_env_args = h5_file["data"].attrs.get("env_args", "{}") try: env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) - except ValueError: - return {} - return env_args if isinstance(env_args, dict) else {} + except (TypeError, ValueError): + env_args = None + if not isinstance(env_args, dict): + raise ValueError(f"{h5_file.filename}: env_args are not a JSON object.") + entry = env_args.get("autodata") + if "autodata" in env_args and not ( + isinstance(entry, dict) + and isinstance(entry.get("run_id"), str) + and isinstance(entry.get("exported_episodes"), int) + ): + raise ValueError(f"{h5_file.filename}: the autodata entry lacks its run_id or exported_episodes.") + return env_args class Episode: diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 7647f16..180bd11 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -13,7 +13,7 @@ import re import textwrap import yaml -from collections.abc import Mapping +from collections.abc import Hashable, Mapping from pathlib import Path from typing import Any @@ -41,7 +41,9 @@ def read_run( another run is set aside and the run is reported from its own attempt log. """ result_path = Path(result_file) - result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None + result = _read_json(result_path) if result_path.exists() else None + if result is not None and not isinstance(result, dict): + raise ValueError(f"{result_file} is not a generation result.") warnings = [] if result_path.exists() else [f"{result_file} does not exist"] if result is not None and run_id is not None and result.get("run", {}).get("run_id") != run_id: warnings.append(f"{result_file} belongs to another run, so the run is reported from its attempt log") @@ -55,10 +57,18 @@ def read_run( attempts = None if attempts_path.exists(): lines = attempts_path.read_text(encoding="utf-8").splitlines() - attempts = [json.loads(line) for line in lines if line.strip()] + attempts = [_read_json(attempts_path, line) for line in lines if line.strip()] return result, attempts, warnings +def _read_json(path: Path, text: str | None = None) -> Any: + """JSON from ``text``, or the whole file; a parse error names the file.""" + try: + return json.loads(path.read_text(encoding="utf-8") if text is None else text) + except ValueError as exc: + raise ValueError(f"{path} is not valid JSON: {exc}") from exc + + def check_same_run(evidence: Evidence) -> list[str]: """Refuse a dataset and run record that come from different runs; return warnings for unverifiable data. @@ -258,21 +268,43 @@ def _check_summary(evidence: Evidence) -> dict[str, Any]: } +_GATE_FIELDS = {"min", "max", "key", "version"} + + +class _GateLoader(yaml.SafeLoader): + """Safe YAML loading that refuses a repeated key, which plain YAML silently resolves to its last value.""" + + def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Any, Any]: + keys = [self.construct_object(key_node, deep=deep) for key_node, _ in node.value] + repeated = sorted({repr(key) for key in keys if isinstance(key, Hashable) and keys.count(key) > 1}) + if repeated: + raise yaml.constructor.ConstructorError(None, None, f"repeated keys {', '.join(repeated)}", node.start_mark) + return super().construct_mapping(node, deep=deep) + + def load_gate(gate_file: str) -> dict[str, dict[str, Any]]: """The gate in ``gate_file``: per metric, optional ``min``/``max`` bounds, a ``key`` into a dict value (default: the metric's primary entry) and a ``version``. Raises: - ValueError: If the file does not map metric names to bounds, or a bound is not a finite number. + ValueError: If the file does not map metric names to these fields, a bound is not a finite number, the key is + not a name or the version not an integer. """ - gate = yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")) + try: + gate = yaml.load(Path(gate_file).read_text(encoding="utf-8"), Loader=_GateLoader) + except (OSError, yaml.YAMLError) as exc: + raise ValueError(f"{gate_file} is not a readable YAML gate: {exc}") from exc if not isinstance(gate, Mapping): raise ValueError(f"{gate_file} must map metric names to bounds, such as 'name: {{min: 0.5}}'.") entries = {} for name, bound in gate.items(): bound = {} if bound is None else bound - if not isinstance(bound, Mapping): - raise ValueError(f"Gate entry {name!r} must map bounds, such as {{min: 0.5}}, not {bound!r}.") + if not isinstance(bound, Mapping) or not set(bound) <= _GATE_FIELDS: + raise ValueError(f"Gate entry {name!r} must map some of {sorted(_GATE_FIELDS)}, not {bound!r}.") + if not isinstance(bound.get("key", ""), str): + raise ValueError(f"Gate entry {name!r} has a key that is not an entry name such as p95: {bound['key']!r}.") + if isinstance(bound.get("version", 1), bool) or not isinstance(bound.get("version", 1), int): + raise ValueError(f"Gate entry {name!r} has a version that is not an integer: {bound['version']!r}.") entries[name] = {**bound, **{side: _bound(name, bound[side]) for side in ("min", "max") if side in bound}} return entries @@ -457,7 +489,7 @@ def _not_computed_because(row: Mapping[str, Any], inputs: Mapping[str, Any]) -> if missing == "source": return "needs --source" if missing.startswith("source."): - return "needs a --source recorded with --record_metric_inputs" + return "needs a --source annotated with --record_metric_inputs" if missing == "attempts" or missing.startswith("run."): return "needs --result_file" if inputs["result_file"] is None else "needs --record_metric_inputs" if missing.startswith("attempts.extras."): diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index c8b57e0..ae700b1 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -110,6 +110,12 @@ def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): with pytest.raises(ValueError, match="not the one the run started from"): build_report(dataset, result_file=result_file, source=_dataset(tmp_path / "source.hdf5"), metrics={}) + for corrupt, reason in (("{'run_id': 'run-a'}", "not a JSON object"), ('{"autodata": null}', "autodata entry")): + with h5py.File(dataset, "r+") as h5_file: + h5_file["data"].attrs["env_args"] = corrupt + with pytest.raises(ValueError, match=reason): + build_report(dataset, result_file=_run(tmp_path, "run-a"), metrics={}) + def test_gate_flags_violations_and_refuses_a_malformed_gate(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} @@ -136,6 +142,11 @@ def test_gate_flags_violations_and_refuses_a_malformed_gate(tmp_path): "absent: {min: .nan}", "absent: {min: false}", "absent: {min: {p95: 1}}", + "absent: {mn: 0.5}", + "gap: {key: [p95]}", + "old: {version: '1'}", + "low: {min: 0.5}\nlow: {max: 1.0}", + "low: {min: 0.5", "absent: 0.5", "[low]", "", @@ -181,7 +192,7 @@ def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp path = tmp_path / "d.hdf5" with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - data.attrs["env_args"] = "{'env_name': 'not JSON'}" + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) odd = data.create_group("demo_0") odd.create_dataset("actions", data=np.zeros((2, 7))) odd.create_dataset("obs/robot/root_pose", data=1.0) diff --git a/autodata_tests/utils/test_validate_dataset.py b/autodata_tests/utils/test_validate_dataset.py index d109227..eee0200 100644 --- a/autodata_tests/utils/test_validate_dataset.py +++ b/autodata_tests/utils/test_validate_dataset.py @@ -54,3 +54,14 @@ def test_report_options_without_report_are_an_error(tmp_path: Path) -> None: with pytest.raises(SystemExit): validate_dataset.main([str(tmp_path / "data.hdf5"), "--gate", str(tmp_path / "gate.yaml")]) + + +def test_report_refused_for_an_unreadable_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """A source the validator cannot read is refused in one line instead of a traceback.""" + + dataset = tmp_path / "data.hdf5" + with h5py.File(dataset, "w") as h5_file: + h5_file.create_group("data/demo_0").create_dataset("actions", data=[[0.0]]) + + assert validate_dataset.main([str(dataset), "--report", "--source", str(tmp_path / "missing.hdf5")]) == 1 + assert "Report refused" in capsys.readouterr().out diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 26c1f5d..c97b8bc 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -33,7 +33,7 @@ The command prints a summary; ``--report_file`` writes the full report, with eve NOT COMPUTED not reported by this algorithm: planning_failure_rate, planning_time - needs a --source recorded with --record_metric_inputs: annotation_pass_rate + needs a --source annotated with --record_metric_inputs: annotation_pass_rate - The second line is the verdict: valid episodes, how many metrics were computed, and, with ``--gate``, the gate, whose metrics are marked ✓ or ✗. @@ -61,7 +61,8 @@ Gating planning_time: {max: 5.0, key: p95} A gated metric also violates the gate if it is not applicable, fails, or reports a different -``version``. A bound that is not a finite number is an error. +``version``. A gate entry may set ``min``, ``max``, ``key`` and ``version``; any other field, or a bound +that is not a finite number, is an error. Episode checks -------------- diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index a2b7a1a..81dfbbc 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -199,7 +199,15 @@ def main(argv: list[str] | None = None) -> int: if args.report and results[0].num_episodes is not None: for module in args.metrics: importlib.import_module(module) - report = build_report(args.files[0], result_file=args.result_file, source=args.source) + source = validate_file(args.source) if args.source else None + if source is not None and source.num_episodes is None: + print(f"\nReport refused: {args.source}: {'; '.join(source.issues)}") + return 1 + try: + report = build_report(args.files[0], result_file=args.result_file, source=args.source) + except ValueError as exc: + print(f"\nReport refused: {exc}") + return 1 results = gate_results(report, gate) if gate is not None else None print() print(format_report(report, results)) From 88446f014eb5148339b16e1d377b8237c30da2da Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 12:52:15 -0500 Subject: [PATCH 23/24] refactor: parse every record the report reads against one declared shape - autodata_metrics/records.py declares the shapes of the autodata entry, the result file and its run block, and an attempt-log line. A record is parsed once where it is read; a missing field, a wrong type, a negative count or a number a float cannot hold is refused with an error naming the file and field, and absent or null optional fields are left out, so readers see a field well formed or not at all. Fields no metric reads are not checked. - Run files that cannot be read, are not UTF-8 or nest too deeply are refused with the file named. - Value checks itself when it is created, so a metric with a malformed Value is an error row and the report reads only well-formed values. - A source merged or edited after annotation has its annotation record set aside, with a warning. - The e2e helpers parse what generation and annotation write, so a producer that drifts from the shapes fails its test. - Generated tests mutate every field of every record and gate with wrong values, and feed unreadable run files, asserting the report refuses or reports them and never raises; a corpus of degenerate episodes asserts no check raises. - Docs: what the report trusts. --- autodata_metrics/evidence.py | 23 +-- autodata_metrics/records.py | 156 ++++++++++++++++ autodata_metrics/registry.py | 12 ++ autodata_metrics/report.py | 78 ++++---- autodata_metrics/validity.py | 2 +- autodata_tests/metrics/test_records.py | 238 ++++++++++++++++++++++++ autodata_tests/metrics/test_report.py | 28 ++- autodata_tests/metrics/test_validity.py | 2 +- autodata_tests/utils/utils.py | 14 +- docs/pages/references/metrics.rst | 10 + 10 files changed, 501 insertions(+), 62 deletions(-) create mode 100644 autodata_metrics/records.py create mode 100644 autodata_tests/metrics/test_records.py diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 5df3727..3180c95 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -12,6 +12,8 @@ from functools import cached_property from typing import Any +from autodata_metrics.records import InvalidRecord, parse_env_args + def _has_path(record: Any, dotted_key: str) -> bool: """Whether ``record[a][b]`` exists for ``dotted_key`` ``"a.b"``.""" @@ -23,27 +25,18 @@ def _has_path(record: Any, dotted_key: str) -> bool: def read_env_args(h5_file: h5py.File) -> dict[str, Any]: - """The dataset's ``env_args``, or empty when it records none. + """The dataset's ``env_args``, with its ``autodata`` entry parsed; empty when it records none. Raises: - ValueError: If ``env_args`` is not a JSON object, or its ``autodata`` entry lacks the run id and exported - episode count; a run cannot be verified against corrupt metadata. + InvalidRecord: If ``env_args`` is not JSON, or it or its entry lacks the shape the metrics read. """ + where = f"{h5_file.filename} env_args" raw_env_args = h5_file["data"].attrs.get("env_args", "{}") try: env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) - except (TypeError, ValueError): - env_args = None - if not isinstance(env_args, dict): - raise ValueError(f"{h5_file.filename}: env_args are not a JSON object.") - entry = env_args.get("autodata") - if "autodata" in env_args and not ( - isinstance(entry, dict) - and isinstance(entry.get("run_id"), str) - and isinstance(entry.get("exported_episodes"), int) - ): - raise ValueError(f"{h5_file.filename}: the autodata entry lacks its run_id or exported_episodes.") - return env_args + except (TypeError, ValueError, RecursionError) as exc: + raise InvalidRecord(f"{where} must be JSON: {exc}") from exc + return parse_env_args(env_args, where) class Episode: diff --git a/autodata_metrics/records.py b/autodata_metrics/records.py new file mode 100644 index 0000000..82be6b7 --- /dev/null +++ b/autodata_metrics/records.py @@ -0,0 +1,156 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shapes of the records a report reads, checked once when a record is read. + +A record that does not match its shape is refused with an error naming the file and the field, so no metric reads a +malformed value. Fields a shape does not list are left unchecked, so producers can add fields that older readers +ignore. +""" + +from __future__ import annotations + +import numbers +import sys +from dataclasses import dataclass +from typing import Any + +from autodata_utils.dataset_metadata import AUTODATA_ENTRY_VERSION + + +class InvalidRecord(ValueError): + """A record without the shape the report reads.""" + + +@dataclass(frozen=True) +class Maybe: + """A field that may be absent or null.""" + + shape: Any + + +@dataclass(frozen=True) +class ListOf: + """A list of values of one shape, of a fixed length when ``length`` is given.""" + + shape: Any + length: int | None = None + + +@dataclass(frozen=True) +class MapOf: + """An object whose values all have one shape.""" + + shape: Any + + +NUMBER = "a finite number" +COUNT = "a non-negative integer" +DURATION = "a non-negative finite number" + +INPUT = {"path": str, "sha256": str} +RUN = { + "run_id": str, + "seed": Maybe(int), + "revision": Maybe(str), + "inputs": Maybe(MapOf(Maybe(INPUT))), + "device": Maybe(str), + "gpu": Maybe(str), + "cpu": Maybe(str), + "num_gpus": Maybe(COUNT), + "num_envs": Maybe(COUNT), + "setup_time_s": Maybe(DURATION), + "generation_time_s": Maybe(DURATION), + "attempts_file": Maybe(str), +} +RESULT = {"num_success": COUNT, "num_failures": Maybe(COUNT), "num_attempts": COUNT, "run": Maybe(RUN)} +ATTEMPT = { + "run_id": str, + "env_id": COUNT, + "success": bool, + "start_time_s": NUMBER, + "end_time_s": NUMBER, + "extras": Maybe({"planning_time_s": Maybe(DURATION), "planning_failed": Maybe(bool)}), +} +ANNOTATION = { + "mode": Maybe(("auto", "manual")), + "input_episodes": COUNT, + "processed_episodes": COUNT, + "annotated_episodes": COUNT, + "setup_time_s": Maybe(DURATION), + "time_s": DURATION, +} +ENTRY = { + "version": COUNT, + "run_id": str, + "exported_episodes": COUNT, + "pose_action": Maybe(MapOf({"action_slice": ListOf(int, 2), "bound": Maybe(NUMBER)})), + "annotation": Maybe(ANNOTATION), +} +ENV_ARGS = {"autodata": Maybe(ENTRY)} + + +def parse(value: Any, shape: Any, where: str) -> Any: + """``value`` checked against ``shape``, with its absent or null optional fields left out. + + A shape is a type, a tuple of allowed values, :data:`NUMBER`, :data:`COUNT`, :data:`DURATION`, a + :class:`Maybe`, :class:`ListOf` or :class:`MapOf`, or a dict of field shapes; a dict's fields are required + unless their shape is a :class:`Maybe`. + + Raises: + InvalidRecord: Naming ``where`` and the field, if ``value`` does not have ``shape``. + """ + if isinstance(shape, Maybe): + return None if value is None else parse(value, shape.shape, where) + if isinstance(shape, dict): + _require(isinstance(value, dict), value, "an object", where) + parsed = dict(value) + for name, field_shape in shape.items(): + _require(name in value or isinstance(field_shape, Maybe), value, f"an object with {name!r}", where) + field = parse(value.get(name), field_shape, f"{where}.{name}") + if field is None: + parsed.pop(name, None) + else: + parsed[name] = field + return parsed + if isinstance(shape, ListOf): + _require(isinstance(value, list) and shape.length in (None, len(value)), value, "a list", where) + return [parse(item, shape.shape, f"{where}[{index}]") for index, item in enumerate(value)] + if isinstance(shape, MapOf): + _require(isinstance(value, dict), value, "an object", where) + parsed = {key: parse(item, shape.shape, f"{where}.{key}") for key, item in value.items()} + return {key: item for key, item in parsed.items() if item is not None} + if isinstance(shape, tuple): + _require(value in shape, value, f"one of {list(shape)}", where) + elif shape in (NUMBER, DURATION): + _require(is_finite_number(value) and (shape is NUMBER or value >= 0), value, shape, where) + elif shape is COUNT: + _require(isinstance(value, int) and not isinstance(value, bool) and value >= 0, value, shape, where) + else: + exact = isinstance(value, shape) and not (shape is int and isinstance(value, bool)) + _require(exact, value, shape.__name__, where) + return value + + +def parse_env_args(env_args: Any, where: str) -> dict[str, Any]: + """A dataset's ``env_args`` with its ``autodata`` entry parsed, refusing an entry from a newer AutoData.""" + entry = env_args.get("autodata") if isinstance(env_args, dict) else None + version = entry.get("version") if isinstance(entry, dict) else None + if isinstance(version, int) and version > AUTODATA_ENTRY_VERSION: + raise InvalidRecord(f"{where}.autodata was written by a newer AutoData (entry version {version}).") + parsed = parse(env_args, ENV_ARGS, where) + annotation = parsed.get("autodata", {}).get("annotation") + if annotation is not None: + counts = [annotation[name] for name in ("annotated_episodes", "processed_episodes", "input_episodes")] + _require(counts == sorted(counts), counts, "annotated <= processed <= input", f"{where}.autodata.annotation") + return parsed + + +def is_finite_number(value: Any) -> bool: + """Whether ``value`` is a real number other than a bool, of a magnitude a float can hold.""" + return isinstance(value, numbers.Real) and not isinstance(value, bool) and abs(value) <= sys.float_info.max + + +def _require(condition: bool, value: Any, expected: str, where: str) -> None: + if not condition: + raise InvalidRecord(f"{where} must be {expected}, not {value!r}.") diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py index c2be456..b076cb5 100644 --- a/autodata_metrics/registry.py +++ b/autodata_metrics/registry.py @@ -10,10 +10,13 @@ from __future__ import annotations +import numbers from collections.abc import Callable, Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal +from autodata_metrics.records import is_finite_number + if TYPE_CHECKING: from autodata_metrics.evidence import Episode, Evidence @@ -38,6 +41,15 @@ class Value: n: int details: Mapping[str, Any] = field(default_factory=dict) + def __post_init__(self) -> None: + values = self.value.values() if isinstance(self.value, Mapping) else [self.value] + if not all(is_finite_number(item) for item in values): + raise ValueError(f"value {self.value!r} is not a finite number or a mapping of finite numbers") + if not isinstance(self.n, numbers.Integral) or self.n < 0: + raise ValueError(f"sample count {self.n!r} is not a non-negative integer") + if not isinstance(self.details, Mapping): + raise ValueError(f"details {self.details!r} are not a mapping") + @dataclass(frozen=True) class MetricSpec: diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 180bd11..8e643d6 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -9,7 +9,6 @@ import h5py import json import math -import numbers import re import textwrap import yaml @@ -18,6 +17,7 @@ from typing import Any from autodata_metrics.evidence import Evidence, read_env_args +from autodata_metrics.records import ATTEMPT, RESULT, InvalidRecord, is_finite_number, parse from autodata_metrics.registry import ( AREAS, REGISTERED_CHECKS, @@ -41,10 +41,9 @@ def read_run( another run is set aside and the run is reported from its own attempt log. """ result_path = Path(result_file) - result = _read_json(result_path) if result_path.exists() else None - if result is not None and not isinstance(result, dict): - raise ValueError(f"{result_file} is not a generation result.") - warnings = [] if result_path.exists() else [f"{result_file} does not exist"] + text = _read_text(result_path) + result = None if text is None else parse(_json(text, result_path), RESULT, result_file) + warnings = [] if text is not None else [f"{result_file} does not exist"] if result is not None and run_id is not None and result.get("run", {}).get("run_id") != run_id: warnings.append(f"{result_file} belongs to another run, so the run is reported from its attempt log") result = None @@ -54,19 +53,32 @@ def read_run( attempts_path = Path(attempts_file_for(result_file, run_id)) else: return result, None, warnings + text = _read_text(attempts_path) attempts = None - if attempts_path.exists(): - lines = attempts_path.read_text(encoding="utf-8").splitlines() - attempts = [_read_json(attempts_path, line) for line in lines if line.strip()] + if text is not None: + attempts = [ + parse(_json(line, attempts_path), ATTEMPT, f"{attempts_path} line {number}") + for number, line in enumerate(text.splitlines(), start=1) + if line.strip() + ] return result, attempts, warnings -def _read_json(path: Path, text: str | None = None) -> Any: - """JSON from ``text``, or the whole file; a parse error names the file.""" +def _read_text(path: Path) -> str | None: + """The file's text, or None when there is no such file.""" + try: + return path.read_text(encoding="utf-8") + except (FileNotFoundError, IsADirectoryError, NotADirectoryError): + return None + except (OSError, UnicodeDecodeError) as exc: + raise InvalidRecord(f"{path} cannot be read: {exc}") from exc + + +def _json(text: str, path: Path) -> Any: try: - return json.loads(path.read_text(encoding="utf-8") if text is None else text) - except ValueError as exc: - raise ValueError(f"{path} is not valid JSON: {exc}") from exc + return json.loads(text) + except (ValueError, RecursionError) as exc: + raise InvalidRecord(f"{path} must be JSON: {exc}") from exc def check_same_run(evidence: Evidence) -> list[str]: @@ -160,9 +172,8 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": str(exc)} except Exception as exc: # A failing metric must not sink the report of every other metric. return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} - problem = _unreportable(value) - if problem is not None: - return {**row, "status": "error", "reason": problem} + if not isinstance(value, Value): + return {**row, "status": "error", "reason": f"returned {type(value).__name__}, not a Value"} row = { **row, "status": "ok", @@ -175,25 +186,6 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return row -def _unreportable(value: Any) -> str | None: - """Why a metric's return value cannot be reported, or None.""" - if not isinstance(value, Value): - return f"returned {type(value).__name__}, not a Value" - if not _is_finite(value.value): - return f"value {value.value!r} is not a finite number or a mapping of finite numbers" - if not isinstance(value.n, numbers.Integral): - return f"sample count {value.n!r} is not an integer" - if not isinstance(value.details, Mapping): - return f"details {value.details!r} are not a mapping" - return None - - -def _is_finite(value: Any) -> bool: - """Whether ``value`` is a finite number, or a mapping of finite numbers.""" - items = value.values() if isinstance(value, Mapping) else [value] - return all(isinstance(item, numbers.Real) and math.isfinite(item) for item in items) - - def _finite_or_none(value: Any) -> Any: """``value`` with every non-finite number replaced by None, so the report stays valid JSON.""" value = value.tolist() if hasattr(value, "tolist") else value @@ -228,6 +220,8 @@ def build_report( run_id = read_env_args(h5_file).get("autodata", {}).get("run_id") result, attempts, warnings = read_run(result_file, run_id) if result_file else (None, None, []) source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None + if source_evidence is not None: + warnings += _set_aside_merged_annotation(source_evidence, source) evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) source_record = input_record(source) @@ -257,6 +251,18 @@ def build_report( } +def _set_aside_merged_annotation(source: Evidence, path: str) -> list[str]: + """Drop the annotation record of a source merged or edited after annotation, since it counts other episodes.""" + entry = source.env_args.get("autodata") + if entry is None or "annotation" not in entry or entry["exported_episodes"] == len(source.all_episodes): + return [] + source.env_args = {**source.env_args, "autodata": {key: item for key, item in entry.items() if key != "annotation"}} + return [ + f"{path} holds {len(source.all_episodes)} episodes but was annotated with {entry['exported_episodes']}, so its" + " annotation record is set aside" + ] + + def _check_summary(evidence: Evidence) -> dict[str, Any]: return { "num_episodes": len(evidence.all_episodes), @@ -315,7 +321,7 @@ def _bound(name: str, limit: Any) -> float: if isinstance(limit, str): with contextlib.suppress(ValueError): number = float(limit) - if not isinstance(number, int | float) or isinstance(number, bool) or not math.isfinite(number): + if not is_finite_number(number): raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limit!r}.") return number diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 04df46a..6fa360e 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -98,7 +98,7 @@ def pose_action_bounds(episode: Episode, ev: Evidence) -> str | None: actions = episode["actions"] bad = [] for eef, layout in ev.env_args["autodata"]["pose_action"].items(): - if layout["bound"] is None: + if layout.get("bound") is None: continue start, end = layout["action_slice"] peak = float(np.abs(actions[:, start:end]).max(initial=0.0)) diff --git a/autodata_tests/metrics/test_records.py b/autodata_tests/metrics/test_records.py new file mode 100644 index 0000000..50bc4f0 --- /dev/null +++ b/autodata_tests/metrics/test_records.py @@ -0,0 +1,238 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generated robustness tests: every malformed record or gate is refused or reported, and no check raises.""" + +import h5py +import json +import numpy as np +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from autodata_metrics import build_report, format_report, gate_results, load_gate +from autodata_metrics import report as report_module +from autodata_metrics.evidence import Evidence +from autodata_metrics.registry import REGISTERED_CHECKS +from autodata_utils.generation_result import input_record + +ABSENT = object() +SOURCE_RECORD = "source record" +"""Stands for the source's path and SHA-256, filled in once the source file is written.""" +WRONG_VALUES = (ABSENT, None, "", "x", -1, 1.5, 10**400, True, float("nan"), [], {}, {"x": 1}) + + +@pytest.fixture(autouse=True) +def _no_git(monkeypatch): + monkeypatch.setattr(report_module, "code_revision", lambda: None) + + +def _write_dataset(path: Path, entry: dict, num_episodes: int = 2) -> str: + pose = np.tile(np.eye(4), (3, 1, 1)) + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", "autodata": entry}) + for index in range(num_episodes): + episode = data.create_group(f"demo_{index}") + episode.create_dataset("actions", data=np.zeros((3, 7))) + episode.create_dataset("step_success", data=[False, True, True]) + episode.create_dataset("obs/datagen_info/eef_pose/arm", data=pose) + episode.create_dataset("obs/datagen_info/target_eef_pose/arm", data=pose) + return str(path) + + +def _samples(tmp_path: Path) -> dict[str, dict]: + """Records as AutoData writes them, for a run that exported two episodes from a source of two.""" + annotation = { + "mode": "auto", + "input_episodes": 2, + "processed_episodes": 2, + "annotated_episodes": 2, + "setup_time_s": 10.0, + "time_s": 20.0, + } + source_entry = {"version": 1, "run_id": "annotation", "exported_episodes": 2, "annotation": annotation} + run = { + "run_id": "run-a", + "seed": 1, + "revision": None, + "inputs": {"source": SOURCE_RECORD, "env_profile": None}, + "device": "cuda:0", + "gpu": "GPU", + "cpu": "CPU", + "num_gpus": 1, + "num_envs": 1, + "setup_time_s": 5.0, + "generation_time_s": 30.0, + "attempts_file": "result.run-a.attempts.jsonl", + } + attempt = { + "run_id": "run-a", + "env_id": 0, + "success": True, + "start_time_s": 0.0, + "end_time_s": 1.0, + "initial_object_positions": {"cube": [0.1, 0.2]}, + "extras": {"planning_time_s": 0.5, "planning_failed": False}, + } + return { + "entry": { + "version": 1, + "run_id": "run-a", + "exported_episodes": 2, + "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0}}, + }, + "source_entry": source_entry, + "result": {"algorithm": "mimicgen", "num_success": 2, "num_failures": 1, "num_attempts": 3, "run": run}, + "attempt": attempt, + } + + +def _paths(record: dict, prefix: tuple = ()) -> Iterator[tuple]: + """Every field path of a record, nested objects included.""" + for key, value in record.items(): + yield (*prefix, key) + if isinstance(value, dict): + yield from _paths(value, (*prefix, key)) + + +def _replaced(record: dict, path: tuple, value: object) -> dict: + copy = json.loads(json.dumps(record)) + parent = copy + for key in path[:-1]: + parent = parent[key] + if value is ABSENT: + del parent[path[-1]] + else: + parent[path[-1]] = value + return copy + + +def _report_or_refusal(tmp_path: Path, records: dict) -> dict | None: + """Build, summarize and serialize the report of these records; None when the report refuses them.""" + dataset = _write_dataset(tmp_path / "data.hdf5", records["entry"]) + source = _write_dataset(tmp_path / "source.hdf5", records["source_entry"]) + result_file = tmp_path / "result.json" + result_file.write_text( + json.dumps(records["result"]).replace(json.dumps(SOURCE_RECORD), json.dumps(input_record(source))) + ) + first = records["attempt"] + lines = [first, {**first, "start_time_s": 1.0, "end_time_s": 2.0}, {**first, "success": False, "start_time_s": 2.0}] + (tmp_path / "result.run-a.attempts.jsonl").write_text("".join(json.dumps(line) + "\n" for line in lines)) + gate = tmp_path / "gate.yaml" + gate.write_text("generation_success_rate: {min: 0.1}\naction_rotation_error: {max: 0.001}\n") + try: + report = build_report(dataset, result_file=str(result_file), source=source) + except ValueError: + return None + json.dumps(report, allow_nan=False) + format_report(report, gate_results(report, load_gate(str(gate)))) + return report + + +def test_every_malformed_record_field_is_refused_or_reported(tmp_path): + samples = _samples(tmp_path) + report = _report_or_refusal(tmp_path, samples) + assert report is not None and report["metrics"]["time_to_dataset"]["status"] == "ok" + unread = {**samples["attempt"], "initial_object_positions": {"cube": [float("nan"), 0.0]}} + assert _report_or_refusal(tmp_path, {**samples, "attempt": unread}) is not None + negative = {**samples["attempt"], "extras": {"planning_time_s": -5.0, "planning_failed": False}} + assert _report_or_refusal(tmp_path, {**samples, "attempt": negative}) is None + crashes = [] + for kind in ("entry", "source_entry", "result", "attempt"): + for path in _paths(samples[kind]): + for wrong in WRONG_VALUES: + records = {**samples, kind: _replaced(samples[kind], path, wrong)} + try: + _report_or_refusal(tmp_path, records) + except Exception as exc: # Anything but a refusal is a crash this test exists to find. + crashes.append(f"{kind}.{'.'.join(path)} = {wrong!r}: {type(exc).__name__}: {exc}") + assert not crashes, "\n".join(crashes[:20]) + + +def test_unreadable_run_files_are_refused(tmp_path): + samples = _samples(tmp_path) + crashes = [] + for name in ("result.json", "result.run-a.attempts.jsonl"): + for raw in (b"", b"{", b"\xff\xfe", b"[" * 10000 + b"]" * 10000): + _report_or_refusal(tmp_path, samples) + (tmp_path / name).write_bytes(raw) + try: + build_report( + str(tmp_path / "data.hdf5"), + result_file=str(tmp_path / "result.json"), + source=str(tmp_path / "source.hdf5"), + ) + except ValueError: + continue + except Exception as exc: # Anything but a refusal is a crash this test exists to find. + crashes.append(f"{name} = {raw[:8]!r}: {type(exc).__name__}: {exc}") + assert not crashes, "\n".join(crashes) + + +def test_every_malformed_gate_field_is_refused_or_evaluated(tmp_path): + samples = _samples(tmp_path) + _report_or_refusal(tmp_path, samples) + report = build_report( + str(tmp_path / "data.hdf5"), result_file=str(tmp_path / "result.json"), source=str(tmp_path / "source.hdf5") + ) + gate = { + "generation_success_rate": {"min": 0.1, "version": 1}, + "action_rotation_error": {"max": 0.001, "key": "p95"}, + } + crashes = [] + for path in _paths(gate): + for wrong in WRONG_VALUES: + gate_file = tmp_path / "gate.yaml" + gate_file.write_text(json.dumps(_replaced(gate, path, wrong))) + try: + loaded = load_gate(str(gate_file)) + except ValueError: + continue + try: + format_report(report, gate_results(report, loaded)) + except Exception as exc: # Evaluating a loaded gate must never raise. + crashes.append(f"{'.'.join(path)} = {wrong!r}: {type(exc).__name__}: {exc}") + assert not crashes, "\n".join(crashes) + + +def test_checks_never_fail_an_episode_by_raising(tmp_path): + steps = 3 + degenerate = { + "scalar": {"obs/scalar": np.float32(1.0), "obs/robot/root_pose": np.float32(1.0)}, + "strings": {"obs/names": np.array([b"a", b"b", b"c"])}, + "empty_camera": {"obs/camera": np.zeros((0, 4, 4, 3), np.uint8)}, + "float_camera": {"obs/depth": np.zeros((steps, 4, 4, 1), np.float32)}, + "gray_camera": {"obs/gray": np.zeros((steps, 4, 4), np.uint8)}, + "zero_pose": {"obs/datagen_info/eef_pose/arm": np.zeros((steps, 4, 4))}, + "short_success": {"step_success": np.zeros(1, bool)}, + "no_actions": {}, + "empty_actions": {"actions": np.zeros((0, 7))}, + } + path = tmp_path / "d.hdf5" + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) + for name, arrays in degenerate.items(): + episode = data.create_group(name) + if name != "no_actions": + episode.create_dataset("actions", data=arrays.pop("actions", np.zeros((steps, 7)))) + episode.create_group("obs/datagen_info/target_eef_pose") + for key, array in arrays.items(): + episode.create_dataset(key, data=array) + + with h5py.File(path, "r") as h5_file: + evidence = Evidence(h5_file) + report_module.run_checks(evidence, REGISTERED_CHECKS) + raised = { + f"{check} on {episode}: {reason}" + for check, failed in evidence.failures.items() + for episode, reason in failed.items() + if re.match(r"^[A-Z]\w*(Error|Exception):", reason) + } + assert not raised, "\n".join(sorted(raised)) + report = build_report(str(path)) + json.dumps(report, allow_nan=False) + format_report(report) diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index ae700b1..e3b2f23 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -36,14 +36,18 @@ def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: def _dataset(path: Path, num_episodes: int = 1, entry: dict | None = None) -> str: with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", **({"autodata": entry} if entry else {})}) + autodata = {"autodata": {"version": 1, **entry}} if entry else {} + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", **autodata}) for index in range(num_episodes): data.create_group(f"demo_{index}").create_dataset("actions", data=[[0.0, 1.0]]) return str(path) def _log(tmp_path: Path, run_id: str, successes: list[bool]) -> None: - lines = [json.dumps({"run_id": run_id, "success": success}) for success in successes] + lines = [ + json.dumps({"run_id": run_id, "env_id": 0, "success": success, "start_time_s": step, "end_time_s": step + 1.0}) + for step, success in enumerate(successes) + ] (tmp_path / f"result.{run_id}.attempts.jsonl").write_text("\n".join(lines) + "\n") @@ -95,6 +99,20 @@ def test_dataset_without_an_entry_is_flagged_only_with_a_run_record(tmp_path): assert "episode count only" in build_report(dataset, result_file=_run(tmp_path, "run-a"), metrics={})["warnings"][0] +def test_a_source_merged_after_annotation_has_its_annotation_set_aside(tmp_path): + annotation = {"input_episodes": 1, "processed_episodes": 1, "annotated_episodes": 1, "time_s": 2.0} + source = _dataset( + tmp_path / "source.hdf5", + num_episodes=3, + entry={"run_id": "a", "exported_episodes": 1, "annotation": annotation}, + ) + + report = build_report(_dataset(tmp_path / "d.hdf5"), source=source) + + assert "set aside" in report["warnings"][0] + assert report["metrics"]["annotation_pass_rate"]["status"] == "not_applicable" + + def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} with pytest.raises(ValueError, match="merged or edited"): @@ -110,7 +128,11 @@ def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): with pytest.raises(ValueError, match="not the one the run started from"): build_report(dataset, result_file=result_file, source=_dataset(tmp_path / "source.hdf5"), metrics={}) - for corrupt, reason in (("{'run_id': 'run-a'}", "not a JSON object"), ('{"autodata": null}', "autodata entry")): + malformed_entry = json.dumps({"autodata": {"version": 1, "run_id": 5, "exported_episodes": 1}}) + for corrupt, reason in ( + ("{'run_id': 'run-a'}", "must be JSON"), + (malformed_entry, r"autodata\.run_id must be str"), + ): with h5py.File(dataset, "r+") as h5_file: h5_file["data"].attrs["env_args"] = corrupt with pytest.raises(ValueError, match=reason): diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index 4af7b45..2633450 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -17,7 +17,7 @@ def _write(path: Path, episodes: list[dict[str, np.ndarray]], entry: dict | None = None) -> str: with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - env_args = {"env_name": "Test-v0", **({"autodata": entry} if entry else {})} + env_args = {"env_name": "Test-v0", **({"autodata": {"version": 1, **entry}} if entry else {})} data.attrs["env_args"] = json.dumps(env_args) for index, arrays in enumerate(episodes): group = data.create_group(f"demo_{index}") diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index f46a0cb..42bd977 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -7,6 +7,7 @@ import json import os +from autodata_metrics.records import ATTEMPT, RESULT, parse, parse_env_args from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess @@ -49,8 +50,8 @@ def read_generation_result(result_file: str) -> tuple[int, int]: def assert_recorded_episodes(dataset_file: str) -> dict: """Assert the per-step recording and the ``autodata`` entry agree with the dataset's episodes. - Every episode has one ``step_success`` value per action, its success attribute equals whether - ``step_success`` ever held, and the entry counts every exported episode. + The entry has the shape the metrics read, every episode has one ``step_success`` value per action, its success + attribute equals whether ``step_success`` ever held, and the entry counts every exported episode. Returns: The dataset's ``autodata`` entry. @@ -58,7 +59,7 @@ def assert_recorded_episodes(dataset_file: str) -> dict: with h5py.File(dataset_file, "r") as h5_file: data = h5_file["data"] - entry = json.loads(data.attrs["env_args"])["autodata"] + entry = parse_env_args(json.loads(data.attrs["env_args"]), dataset_file)["autodata"] assert entry["exported_episodes"] == len(data), f"The entry counts {entry['exported_episodes']} episodes." for name, episode in data.items(): step_success = episode["step_success"][:] @@ -68,7 +69,8 @@ def assert_recorded_episodes(dataset_file: str) -> dict: def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: - """Assert a generation run's dataset, result JSON and attempt log describe the same run. + """Assert a generation run's dataset, result JSON and attempt log have the shapes the metrics read and describe + the same run. Returns: The attempt-log lines. @@ -76,10 +78,10 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: entry = assert_recorded_episodes(dataset_file) with open(result_file, encoding="utf-8") as result_handle: - result = json.load(result_handle) + result = parse(json.load(result_handle), RESULT, result_file) attempts_file = os.path.join(os.path.dirname(result_file), result["run"]["attempts_file"]) with open(attempts_file, encoding="utf-8") as attempts_handle: - attempts = [json.loads(line) for line in attempts_handle if line.strip()] + attempts = [parse(json.loads(line), ATTEMPT, attempts_file) for line in attempts_handle if line.strip()] assert entry["exported_episodes"] == result["num_success"], "The dataset and result count different successes." assert len(attempts) == result["num_attempts"], "The attempt log and result count different attempts." assert ( diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index c97b8bc..e4d451e 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -50,6 +50,16 @@ attempt log. The report refuses a dataset, result file and attempt log whose epi counts disagree, as they do after datasets are merged. When the result file records the run's source, the report also refuses a different ``--source``; without that record, the source is not verified. +What the report trusts +---------------------- + +The report checks every record it reads against the shape it expects before computing anything: the +dataset's ``env_args`` and ``autodata`` entry, the result file, each attempt-log line, and the gate. A +record with a missing field, a wrong type or a negative count is refused with a message naming the file +and the field, so no metric is computed from part of a malformed record. Fields the report does not read +are not checked. A metric loaded with ``--metrics`` runs as trusted code, but its ``Value`` is checked when +it is created: a malformed one makes that metric an error, and the other metrics are still computed. + Gating ------ From 1cc004c743b14dbe79f23892bbe8ba81b97dda89 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 29 Oct 2026 21:13:51 -0500 Subject: [PATCH 24/24] fix: check subtask signals and pose shapes, and time frozen frames - no_frozen_frames: a camera counts as frozen when it repeats one frame for 1 s of recorded time, converted with the dataset's dt and decimation, so the threshold no longer depends on the control rate. A dataset that records no control step skips the check. - aligned_lengths: subtask signals must hold one entry per action. Each signal is counted by its entries, since sources store it as a (steps,) column or a (1, steps) row. - valid_rotations: an observed pose not shaped (steps, 4, 4), or a root pose not shaped (steps, 7), fails the episode instead of being skipped. --- autodata_metrics/records.py | 12 +++--- autodata_metrics/validity.py | 56 ++++++++++++++++--------- autodata_tests/metrics/test_report.py | 4 +- autodata_tests/metrics/test_validity.py | 34 +++++++++++---- 4 files changed, 72 insertions(+), 34 deletions(-) diff --git a/autodata_metrics/records.py b/autodata_metrics/records.py index 82be6b7..35f9e56 100644 --- a/autodata_metrics/records.py +++ b/autodata_metrics/records.py @@ -47,6 +47,7 @@ class MapOf: NUMBER = "a finite number" COUNT = "a non-negative integer" DURATION = "a non-negative finite number" +POSITIVE = "a positive finite number" INPUT = {"path": str, "sha256": str} RUN = { @@ -87,14 +88,14 @@ class MapOf: "pose_action": Maybe(MapOf({"action_slice": ListOf(int, 2), "bound": Maybe(NUMBER)})), "annotation": Maybe(ANNOTATION), } -ENV_ARGS = {"autodata": Maybe(ENTRY)} +ENV_ARGS = {"autodata": Maybe(ENTRY), "sim_args": Maybe({"dt": Maybe(POSITIVE), "decimation": Maybe(POSITIVE)})} def parse(value: Any, shape: Any, where: str) -> Any: """``value`` checked against ``shape``, with its absent or null optional fields left out. - A shape is a type, a tuple of allowed values, :data:`NUMBER`, :data:`COUNT`, :data:`DURATION`, a - :class:`Maybe`, :class:`ListOf` or :class:`MapOf`, or a dict of field shapes; a dict's fields are required + A shape is a type, a tuple of allowed values, :data:`NUMBER`, :data:`COUNT`, :data:`DURATION`, :data:`POSITIVE`, + a :class:`Maybe`, :class:`ListOf` or :class:`MapOf`, or a dict of field shapes; a dict's fields are required unless their shape is a :class:`Maybe`. Raises: @@ -122,8 +123,9 @@ def parse(value: Any, shape: Any, where: str) -> Any: return {key: item for key, item in parsed.items() if item is not None} if isinstance(shape, tuple): _require(value in shape, value, f"one of {list(shape)}", where) - elif shape in (NUMBER, DURATION): - _require(is_finite_number(value) and (shape is NUMBER or value >= 0), value, shape, where) + elif shape in (NUMBER, DURATION, POSITIVE): + in_range = is_finite_number(value) and (shape is NUMBER or value > 0 or (shape is DURATION and value == 0)) + _require(in_range, value, shape, where) elif shape is COUNT: _require(isinstance(value, int) and not isinstance(value, bool) and value >= 0, value, shape, where) else: diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index 6fa360e..bf48171 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -6,6 +6,7 @@ from __future__ import annotations import h5py +import math import numpy as np from autodata_metrics.evidence import Episode, Evidence @@ -15,12 +16,14 @@ """Largest deviation from a proper rotation, or from a unit quaternion, a recorded pose may show.""" BLACK_FRAME_MEAN = 2.0 """Mean pixel value, out of 255, below which a camera frame counts as black.""" -FROZEN_FRAMES = 10 -"""Identical consecutive camera frames that count as a frozen stream.""" +FROZEN_SECONDS = 1.0 +"""Recorded time over which a camera that repeats one frame counts as frozen.""" FRAME_CHUNK = 64 """Steps read at a time, so checks of camera frames and other large arrays stream instead of loading whole episodes.""" -PER_EPISODE_PATHS = ("initial_state", "obs/datagen_info/subtask_term_signals", "obs/datagen_info/subtask_start_signals") -"""Paths whose arrays are not one entry per step: the initial state, and annotation's subtask signals.""" +PER_EPISODE_PATHS = ("initial_state",) +"""Paths whose arrays are not one entry per step.""" +SUBTASK_SIGNAL_PATHS = ("obs/datagen_info/subtask_term_signals/", "obs/datagen_info/subtask_start_signals/") +"""Annotation's per-step subtask signals, stored as a ``(steps,)`` column or a ``(1, steps)`` row.""" OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") """Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" @@ -54,14 +57,20 @@ def aligned_lengths(episode: Episode, ev: Evidence) -> str | None: bad = [ path for path, dataset in episode.datasets() - if dataset.ndim > 0 - and not path.startswith(PER_EPISODE_PATHS) + if not path.startswith(PER_EPISODE_PATHS) and path not in ev.camera_paths - and len(dataset) != num_steps + and _steps_held(path, dataset) not in (None, num_steps) ] return f"{', '.join(bad)} do not hold {num_steps} steps" if bad else None +def _steps_held(path: str, dataset: h5py.Dataset) -> int | None: + """Steps ``dataset`` holds: a subtask signal counts its entries in either layout; a scalar holds none.""" + if path.startswith(SUBTASK_SIGNAL_PATHS): + return dataset.size + return len(dataset) if dataset.ndim else None + + def _rotation_errors(poses: np.ndarray) -> np.ndarray: """Per 4x4 pose, how far its rotation is from a proper one: the larger of the largest entry of ``|R^T R - I|`` and ``|det R - 1|``, which also catches reflections.""" @@ -72,20 +81,26 @@ def _rotation_errors(poses: np.ndarray) -> np.ndarray: @check("valid_rotations") def valid_rotations(episode: Episode, ev: Evidence) -> str | None: - """Fails an episode with an observed pose rotation that is not a proper rotation or a root quaternion that is not unit. + """Fails an episode with an observed pose rotation that is not a proper rotation, a root quaternion that is not + unit, or a pose array not shaped ``(steps, 4, 4)`` or ``(steps, 7)``. Both tests hold for either quaternion order, so the check needs no convention. """ bad, checked = [], False for path, dataset in episode.datasets(): - if path.startswith(OBSERVED_POSE_PATHS) and dataset.shape[-2:] == (4, 4): - error = _rotation_errors(dataset[()]).max(initial=0.0) - elif path.endswith("root_pose") and dataset.shape[-1:] == (7,): - error = np.abs(np.linalg.norm(dataset[()][..., 3:7], axis=-1) - 1.0).max(initial=0.0) + if path.startswith(OBSERVED_POSE_PATHS): + shape = (4, 4) + elif path.endswith("/root_pose"): + shape = (7,) else: continue checked = True - if error > ROTATION_TOLERANCE: + if dataset.shape[1:] != shape: + bad.append(f"{path} of shape {dataset.shape}") + continue + poses = dataset[()] + error = _rotation_errors(poses) if shape == (4, 4) else np.abs(np.linalg.norm(poses[:, 3:7], axis=-1) - 1.0) + if error.max(initial=0.0) > ROTATION_TOLERANCE: bad.append(path) if not checked: raise NotApplicable("no recorded rotations") @@ -130,16 +145,19 @@ def no_black_frames(episode: Episode, ev: Evidence) -> str | None: return None -@check("no_frozen_frames", group="image") +@check("no_frozen_frames", group="image", requires=["env_args.sim_args.dt", "env_args.sim_args.decimation"]) def no_frozen_frames(episode: Episode, ev: Evidence) -> str | None: - """Fails an episode whose camera repeats one frame :data:`FROZEN_FRAMES` times in a row.""" - for path, dataset in _image_streams(episode, ev).items(): + """Fails an episode whose camera repeats one frame for :data:`FROZEN_SECONDS`; needs the recorded control step.""" + streams = _image_streams(episode, ev) + step = ev.env_args["sim_args"]["dt"] * ev.env_args["sim_args"]["decimation"] + frozen_frames = max(2, math.ceil(FROZEN_SECONDS / step)) + for path, dataset in streams.items(): previous, repeats = None, 1 for start in range(0, len(dataset), FRAME_CHUNK): for offset, frame in enumerate(dataset[start : start + FRAME_CHUNK]): repeats = repeats + 1 if previous is not None and np.array_equal(frame, previous) else 1 - if repeats >= FROZEN_FRAMES: - return f"{path} repeats one frame {FROZEN_FRAMES} times up to step {start + offset}" + if repeats >= frozen_frames: + return f"{path} repeats one frame for {FROZEN_SECONDS:g} s up to step {start + offset}" previous = frame return None @@ -180,7 +198,7 @@ def correctness_pass_rate(ev: Evidence) -> Value: better="higher", ) def image_stream_pass_rate(ev: Evidence) -> Value: - """Share of episodes whose camera streams hold one frame per step, no black frame and no frozen run.""" + """Share of episodes that pass every image check that applies: one frame per step, no black or frozen frames.""" return _pass_rate(ev, "image", "no camera streams") diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index e3b2f23..88c2a92 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -220,6 +220,7 @@ def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp odd.create_dataset("obs/robot/root_pose", data=1.0) odd.create_group("obs/datagen_info/target_eef_pose") data.create_group("demo_1").create_dataset("actions", data=np.zeros((0, 7))) + data.create_group("demo_2").create_dataset("actions", data=np.zeros((2, 7))) metrics = { "action_rotation_error": REGISTERED_METRICS["action_rotation_error"], "open": _spec("open", lambda ev: Value(0.5, n=2, details={"interval_95": [0.1, float("inf")]})), @@ -235,7 +236,8 @@ def test_degenerate_inputs_are_reported_without_breaking_the_summary_or_json(tmp summary = format_report(report) json.dumps(report, allow_nan=False) - assert report["num_valid_episodes"] == 1 and "aligned_lengths" in report["check_failures"] + failures = {name: entry["failed"] for name, entry in report["check_failures"].items() if entry["failed"]} + assert report["num_valid_episodes"] == 1 and failures == {"aligned_lengths": 1, "valid_rotations": 1} assert report["metrics"]["action_rotation_error"]["status"] == "not_applicable" assert re.search(r"open +50% +2 samples$", summary, re.M) and re.search(r"partial +p50 100%", summary) assert re.search(r"almost +>99.9%", summary) diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index 2633450..ab7a045 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -5,19 +5,21 @@ import h5py import json +import math import numpy as np from pathlib import Path import pytest from autodata_metrics import REGISTERED_METRICS, build_report -from autodata_metrics.validity import FROZEN_FRAMES +from autodata_metrics.validity import FROZEN_SECONDS def _write(path: Path, episodes: list[dict[str, np.ndarray]], entry: dict | None = None) -> str: with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - env_args = {"env_name": "Test-v0", **({"autodata": {"version": 1, **entry}} if entry else {})} + env_args = {"env_name": "Test-v0", "sim_args": {"dt": 0.01, "decimation": 5}} + env_args.update({"autodata": {"version": 1, **entry}} if entry else {}) data.attrs["env_args"] = json.dumps(env_args) for index, arrays in enumerate(episodes): group = data.create_group(f"demo_{index}") @@ -54,6 +56,16 @@ def test_failing_episodes_are_counted_and_left_out_of_dataset_metrics(tmp_path): } +def test_aligned_lengths_counts_subtask_signal_entries_in_either_layout(tmp_path): + signal = "obs/datagen_info/subtask_term_signals/grasp" + column, row, short = np.array([False, True, True]), np.array([[False, True, True]]), np.array([False, True]) + episodes = [{"actions": np.zeros((3, 7)), signal: flags} for flags in (column, row, short)] + + report = build_report(_write(tmp_path / "d.hdf5", episodes), metrics={}) + + assert sorted(report["check_failures"]["aligned_lengths"]["examples"]) == ["demo_2"] + + def test_rotation_and_bound_checks_flag_their_defects(tmp_path): pose = np.tile(np.eye(4), (3, 1, 1)) entry = {"run_id": "r", "exported_episodes": 3, "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0}}} @@ -62,17 +74,19 @@ def test_rotation_and_bound_checks_flag_their_defects(tmp_path): skewed = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose * [[1.1], [1.0], [1.0], [1.0]]} zero = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": np.zeros((3, 4, 4))} reflected = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose * [[-1.0], [1.0], [1.0], [1.0]]} - episodes = [clipped, beyond_bound, skewed, zero, reflected] + flat = {"actions": np.zeros((3, 7)), "obs/datagen_info/eef_pose/arm": pose.reshape(3, 16)} + episodes = [clipped, beyond_bound, skewed, zero, reflected, flat] entry["exported_episodes"] = len(episodes) report = build_report(_write(tmp_path / "d.hdf5", episodes, entry), metrics={}) - failures = {name: sorted(entry["examples"]) for name, entry in report["check_failures"].items() if entry["failed"]} - assert failures == {"pose_action_bounds": ["demo_1"], "valid_rotations": ["demo_2", "demo_3", "demo_4"]} + failures = {name: entry["failed"] for name, entry in report["check_failures"].items() if entry["failed"]} + assert failures == {"pose_action_bounds": 1, "valid_rotations": 4} def test_image_checks_flag_black_frozen_and_missing_frames(tmp_path): - steps = FROZEN_FRAMES + 2 + frozen_frames = math.ceil(FROZEN_SECONDS / (0.01 * 5)) # the control step _write records + steps = frozen_frames + 2 rng = np.random.default_rng(0) def frames(count: int) -> np.ndarray: @@ -81,9 +95,11 @@ def frames(count: int) -> np.ndarray: black = frames(steps) black[1] = 0 frozen = np.repeat(frames(1), steps, axis=0) + paused = frames(steps) + paused[1 : frozen_frames - 1] = paused[0] # Still for just under FROZEN_SECONDS, which is not frozen. episodes = [ {"actions": np.zeros((steps, 7)), "obs/camera": camera} - for camera in (frames(steps), black, frozen, frames(steps - 1)) + for camera in (frames(steps), black, frozen, frames(steps - 1), paused) ] episodes.append({"actions": np.zeros((steps, 7))}) @@ -93,9 +109,9 @@ def frames(count: int) -> np.ndarray: assert failures == { "no_black_frames": ["demo_1"], "no_frozen_frames": ["demo_2"], - "one_frame_per_step": ["demo_3", "demo_4"], + "one_frame_per_step": ["demo_3", "demo_5"], } - assert report["metrics"]["image_stream_pass_rate"]["value"] == pytest.approx(1 / 5) + assert report["metrics"]["image_stream_pass_rate"]["value"] == pytest.approx(2 / 6) def test_final_success_rate_reports_the_source_baseline(tmp_path):