From 82aea57c4399631303e92c3dbb030b380e0537d9 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 28 Oct 2026 20:01:50 -0400 Subject: [PATCH 1/2] Simplify camera processor plumbing and correct env_mask docs Keep each request's data in its renderer group instead of re-indexing the request list, build the processor buffers once with the capture mask, and validate the reserved env_mask output name in CameraCfg.validate_config so the data-type helper only computes. Test the reserved name and processor-input validation through validate(CameraCfg) instead of the private helper, and assert the mask values and call count rather than only their length. Renderers draw every environment on each capture, so describe env_mask as the environments due for the capture rather than rows that still hold the previous one. Merge the two added changelog bullets for the feature. --- docs/source/concepts/sensors/camera.rst | 11 ++++--- .../camera-tensor-processors.added.rst | 7 ++--- .../isaaclab/renderers/render_context.py | 14 ++++----- .../isaaclab/sensors/camera/camera.py | 10 +++--- .../isaaclab/sensors/camera/camera_cfg.py | 18 +++++------ .../renderers/test_camera_output_contract.py | 31 ++++++++----------- 6 files changed, 43 insertions(+), 48 deletions(-) diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 74147b3485b..ed633619371 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -311,11 +311,12 @@ Camera tensor processors renderer for the inputs no earlier processor produces, constructs each processor at the first published capture with its configuration and the named camera buffers, and runs the processors in list order on every new capture. Their outputs are added to :attr:`~sensors.CameraData.output`; a processor can also replace a render output it -reads. Camera resets reach the processors' ``reset`` with the reset environments, and camera teardown their +reads. Camera resets call each processor's ``reset`` with the reset environments, and camera teardown calls its ``close``. .. code-block:: python + from isaaclab.utils import replace from isaaclab.utils.tensor_processors import TensorProcessorCfg depth_camera = replace( @@ -329,10 +330,10 @@ reads. Camera resets reach the processors' ``reset`` with the reset environments Here the renderer produces only ``distance_to_image_plane``, and ``camera.data.output["shaded"]`` holds the processor's result. -With ``update_period > 0``, environments reset at different times and capture on different steps, so a call -can mix new rows with rows that still hold the previous capture. Stateful processors (frame stacking, -temporal filters, video generation) can list ``"env_mask"`` in their ``inputs`` to receive a boolean mask of -the environments with a new capture. The name is reserved, and the mask is not published as an output. +With ``update_period > 0``, environments that reset at different times become due on different steps, but each +capture renders every environment. Stateful processors (frame stacking, temporal filters, video generation) can +list ``"env_mask"`` in their ``inputs`` to receive a boolean mask of the environments due for the capture, whose +frame and timestamp advance. The name is reserved, and the mask is not published as an output. Post-render image signal processing ----------------------------------- diff --git a/source/isaaclab/changelog.d/camera-tensor-processors.added.rst b/source/isaaclab/changelog.d/camera-tensor-processors.added.rst index ab90db127cf..48892f19a59 100644 --- a/source/isaaclab/changelog.d/camera-tensor-processors.added.rst +++ b/source/isaaclab/changelog.d/camera-tensor-processors.added.rst @@ -1,5 +1,4 @@ * Added ``CameraCfg.tensor_processors``, which compute camera outputs from render outputs and earlier - processor outputs after each published capture, and reset and close with the camera. -* Added the reserved ``env_mask`` input for ``CameraCfg.tensor_processors``, a boolean mask of the environments - with a new capture, so stateful processors can skip rows that still hold the previous capture when - ``update_period`` staggers captures. + processor outputs after each published capture, and reset and close with the camera. Processors can read + the reserved ``env_mask`` input, which marks the environments due for each capture when ``update_period`` + staggers them. diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index d02d26cb973..3e6df6e572d 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -365,15 +365,15 @@ def render_into_cameras( self.update_scene_state(physics_step_count) - groups: dict[int, tuple[BaseRenderer, list[int]]] = {} - for index, (renderer, _, _) in enumerate(requests): - groups.setdefault(id(renderer), (renderer, []))[1].append(index) + groups: dict[int, tuple[BaseRenderer, list[tuple[int, Any, CameraData]]]] = {} + for index, (renderer, render_data, camera_data) in enumerate(requests): + groups.setdefault(id(renderer), (renderer, []))[1].append((index, render_data, camera_data)) published = [False] * len(requests) - for renderer, indices in groups.values(): - renderer.render_batch([requests[index][1] for index in indices]) - for index in indices: - published[index] = renderer.read_output(*requests[index][1:]) + for renderer, cameras in groups.values(): + renderer.render_batch([render_data for _, render_data, _ in cameras]) + for index, render_data, camera_data in cameras: + published[index] = renderer.read_output(render_data, camera_data) return published def reset_stage_prepare_flag(self) -> None: diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index edd4aec7e61..c5b3b7f2a0d 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -799,14 +799,14 @@ def _apply_tensor_processors(self) -> None: """Run the processors in order on a newly published capture and publish their outputs.""" if not self.cfg.tensor_processors: return - buffers = dict(self._render_outputs) - # The capture mask is an input for processors, not a camera output. - mask = {CAPTURE_MASK_NAME: ProxyArray(self._is_outdated)} + buffers = {**self._render_outputs, CAPTURE_MASK_NAME: ProxyArray(self._is_outdated)} for index, processor_cfg in enumerate(self.cfg.tensor_processors): # Construction needs the outputs of earlier processors, which exist after their first call. if index == len(self._tensor_processors): - self._tensor_processors.append(processor_cfg.func(processor_cfg, {**buffers, **mask})) - buffers.update(self._tensor_processors[index]({**buffers, **mask})) + self._tensor_processors.append(processor_cfg.func(processor_cfg, buffers)) + buffers.update(self._tensor_processors[index](buffers)) + # The capture mask is an input for processors, not a camera output. + del buffers[CAPTURE_MASK_NAME] self._data.output.update(buffers) """ diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 26bdee7bfbe..3fc449b19f1 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -239,12 +239,12 @@ class OffsetCfg: published capture with the configuration and the named camera buffers. After each published capture, processors run in order on the render outputs and earlier processors' outputs, and their outputs are added to :attr:`CameraData.output`. The renderer produces every input that no earlier processor produces, so a processor - can filter an output in place. Camera resets and teardown reach the processors' ``reset`` and ``close``. + can replace an output it reads. Camera resets and teardown call the processors' ``reset`` and ``close``. - With ``update_period > 0``, environments capture on different steps, so a call can mix new rows with rows that - still hold the previous capture. A stateful processor can list ``"env_mask"`` in its ``inputs`` to receive a - boolean mask of shape ``(N,)`` marking the environments with a new capture. The name is reserved and the mask - is not published as a camera output. + With ``update_period > 0``, environments become due on different steps, but each capture renders every + environment. A stateful processor can list ``"env_mask"`` in its ``inputs`` to receive a boolean mask of shape + ``(N,)`` marking the environments due for the capture, whose frame and timestamp advance. The name is reserved + and the mask is not published as a camera output. """ def __post_init__(self): @@ -277,7 +277,9 @@ def __post_init__(self): setattr(self, field_name, default) def validate_config(self) -> None: - """Validate the requested data types against the selected renderer contract.""" + """Validate processor output names and the requested data types against the selected renderer contract.""" + if any(CAPTURE_MASK_NAME in processor_cfg.outputs for processor_cfg in self.tensor_processors): + raise ValueError(f"Processor output name {CAPTURE_MASK_NAME!r} is reserved for the capture mask.") if self.renderer_cfg is None: return supported_specs = self.renderer_cfg.supported_output_types() @@ -294,15 +296,13 @@ def validate_config(self) -> None: CAPTURE_MASK_NAME = "env_mask" -"""Processor input holding the boolean mask of environments with a new capture.""" +"""Processor input holding the boolean mask of environments due for the capture.""" def _render_data_types(cfg: CameraCfg) -> list[str]: """Return the data types that no processor produces and the inputs that no earlier processor produces.""" produced, inputs = set(), [] for processor_cfg in cfg.tensor_processors: - if CAPTURE_MASK_NAME in processor_cfg.outputs: - raise ValueError(f"Processor output name {CAPTURE_MASK_NAME!r} is reserved for the capture mask.") inputs += [name for name in processor_cfg.inputs if name not in produced and name != CAPTURE_MASK_NAME] produced.update(processor_cfg.outputs) return list(dict.fromkeys([name for name in cfg.data_types if name not in produced] + inputs)) diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 90f35b0f36f..a47e3f9bb98 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -163,29 +163,14 @@ def test_camera_tensor_processors_chain_and_replace_render_outputs(device): # Later captures read the new render rather than the published replacement. for _ in range(2): np.testing.assert_allclose(capture()[:, 0, 0], [2 * background] * 2, atol=1e-6) + # env_mask reaches each processor once per capture with one flag per environment, and is not published. assert set(camera.data.output) == set(camera.data.info) == {"rgb_hdr", "half"} - assert "env_mask" not in camera.data.output and "env_mask" not in camera.data.info - # Processors listing env_mask receive one flag per environment, and it is not published as an output. - assert _ScaleProcessor.masks and all(len(mask) == 2 for mask in _ScaleProcessor.masks) + assert _ScaleProcessor.masks == [[True, True]] * 4 camera.reset(env_ids=[1]) assert _ScaleProcessor.events == [("reset", "half", [1]), ("reset", "rgb_hdr", [1])] assert _ScaleProcessor.events[2:] == [("close", "half"), ("close", "rgb_hdr")] -def test_capture_mask_is_not_requested_from_the_renderer(): - """The camera supplies env_mask to processors, so it never becomes a renderer data type or a processor output.""" - from isaaclab.sensors.camera.camera_cfg import _render_data_types - - def cfg(processors): - return SimpleNamespace(data_types=["rgb_hdr"], tensor_processors=processors) - - stateful = TensorProcessorCfg(func=_ScaleProcessor, inputs=["rgb_hdr", "env_mask"], outputs=["half"]) - assert _render_data_types(cfg([stateful])) == ["rgb_hdr"] - reserved = TensorProcessorCfg(func=_ScaleProcessor, inputs=["rgb_hdr"], outputs=["env_mask"]) - with pytest.raises(ValueError, match="reserved"): - _render_data_types(cfg([reserved])) - - @pytest.mark.parametrize("background", [None, (0.25, 0.5, 0.75)]) @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) def test_newton_hdr_background_preserves_unlit_geometry(background, device): @@ -322,7 +307,7 @@ def test_camera_cfg_rejects_outputs_unsupported_by_renderer(): pytest.importorskip("isaaclab_newton") from isaaclab_newton.renderers import NewtonWarpRendererCfg - def make_cfg(data_type: str) -> CameraCfg: + def make_cfg(data_type: str, *processors: TensorProcessorCfg) -> CameraCfg: return CameraCfg( height=64, width=64, @@ -330,11 +315,21 @@ def make_cfg(data_type: str) -> CameraCfg: spawn=_SPAWN, data_types=[data_type], renderer_cfg=NewtonWarpRendererCfg(), + tensor_processors=list(processors), ) + def processor(source: str, output: str) -> TensorProcessorCfg: + return TensorProcessorCfg(func=_ScaleProcessor, inputs=[source, "env_mask"], outputs=[output]) + validate(make_cfg("rgb_hdr")) with pytest.raises(ValueError, match="simple_shading_full_mdl"): validate(make_cfg("simple_shading_full_mdl")) + # Processor outputs and env_mask are not requested from the renderer, but processor inputs are. + validate(make_cfg("half", processor("rgb_hdr", "half"))) + with pytest.raises(ValueError, match="simple_shading_full_mdl"): + validate(make_cfg("half", processor("simple_shading_full_mdl", "half"))) + with pytest.raises(ValueError, match="reserved"): + validate(make_cfg("rgb_hdr", processor("rgb_hdr", "env_mask"))) @pytest.mark.parametrize("colorize", [True, False]) From c3cceca53ee7e38ccef9659e70ca94d68c119ac5 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 28 Oct 2026 20:01:58 -0400 Subject: [PATCH 2/2] Count a None read_output return as a published capture Renderers written before read_output returned a flag return None. Treating that as unpublished silently skipped every tensor processor on those renderers, leaving processor outputs missing from CameraData.output. Only renderers that hold back a capture need to return False. --- .../changelog.d/camera-tensor-processors.changed.rst | 3 ++- source/isaaclab/isaaclab/renderers/base_renderer.py | 1 + source/isaaclab/isaaclab/renderers/render_context.py | 2 +- .../test/renderers/test_simulation_render_context.py | 5 +++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/isaaclab/changelog.d/camera-tensor-processors.changed.rst b/source/isaaclab/changelog.d/camera-tensor-processors.changed.rst index 5cbbacdb396..0390749134a 100644 --- a/source/isaaclab/changelog.d/camera-tensor-processors.changed.rst +++ b/source/isaaclab/changelog.d/camera-tensor-processors.changed.rst @@ -1,2 +1,3 @@ * Changed ``BaseRenderer.read_output``, ``RenderContext.render_into_camera``, and ``render_into_cameras`` to - return whether a capture was published. Custom renderers should return ``True`` after publishing. + return whether a capture was published. Renderers that hold back a capture return ``False``; a ``None`` + return counts as published. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index 0f0c5cd67f8..475fcfc869a 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -221,6 +221,7 @@ def read_output(self, render_data: Any, camera_data: CameraData) -> bool: Returns: Whether a new capture was published; False when the outputs hold an already-published one. + A None return counts as published. """ pass diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index 3e6df6e572d..b2a4432a810 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -373,7 +373,7 @@ def render_into_cameras( for renderer, cameras in groups.values(): renderer.render_batch([render_data for _, render_data, _ in cameras]) for index, render_data, camera_data in cameras: - published[index] = renderer.read_output(render_data, camera_data) + published[index] = renderer.read_output(render_data, camera_data) is not False return published def reset_stage_prepare_flag(self) -> None: diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 8e6b25a5722..bb63824f0f9 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -243,13 +243,14 @@ def test_render_into_cameras_groups_renderers_and_reads_each_output(sim): first.render_batch.assert_not_called() second.render_batch.assert_not_called() first.read_output.side_effect = [True, False] - second.read_output.return_value = True + second.read_output.return_value = None published = sim.render_context.render_into_cameras( [(first, data[0], cameras[0]), (second, data[1], cameras[1]), (first, data[2], cameras[2])], physics_step_count=1, ) - assert published == [True, True, False] # request order, not renderer-group order + # Request order, not renderer-group order; a renderer returning None still publishes. + assert published == [True, True, False] first.render_batch.assert_called_once_with([data[0], data[2]]) second.render_batch.assert_called_once_with([data[1]]) assert first.read_output.call_args_list == [call(data[0], cameras[0]), call(data[2], cameras[2])]