Python(feat): warn on closed step and rollup status at closure

python/lib/sift_client/_tests/pytest_plugin/_step_status_capture.py

@@ -160,6 +160,11 @@ def test_step(name: str) -> CapturedStep | None:
     return min(matches, key=lambda s: s.step_path.count("."))
 
 
+def module_step() -> CapturedStep:
+    """The outermost step in the run: the module parent, shallowest by path depth."""
+    return min(_steps().values(), key=lambda s: s.step_path.count("."))
+
+
 def final_status(name: str) -> TestStatus | None:
     step = test_step(name)
     return step.statuses[-1] if step and step.statuses else None

python/lib/sift_client/_tests/pytest_plugin/step_status_states.md

@@ -88,6 +88,9 @@ be traced back to its row here without rereading the scenario:
 | `API-07` | `pytest.exit()` after a failed `report_outcome`, in a class | `step.report_outcome(..., False); pytest.exit(...)`            | Failure stop: leaf `ABORTED`; class and module roll up `FAILED`; the failed substep keeps `FAILED`                          |
 | `API-08` | Session-aborting `KeyboardInterrupt` in a class | `step.report_outcome(..., False); raise KeyboardInterrupt`               | System stop: leaf, class, and module all `ABORTED` (the run was cut off, not failed); the failed substep keeps `FAILED`     |
 | `API-09` | Sift `abort()` helper in a class | `from sift_client.pytest_plugin import abort; abort(...)`                          | System stop: leaf, class, module, and the report all `ABORTED`, even with no failing check                                  |
+| `API-10` | Failing measurement recorded after the substep's block exited | `with step.substep(...) as s: pass` then `s.measure(... out-of-bounds)` | `FAILED` on the closed substep and on every ancestor: the late failure is corrected in place rather than reaching only the report |
+| `API-11` | Passing measurement recorded after the substep's block exited | `with step.substep(...) as s: pass` then `s.measure(... in-bounds)`     | Statuses unchanged (`PASSED`); the mis-ordering still warns, since nothing distinguishes it from `API-10` until a value goes out of bounds |
+| `API-12` | `substep` opened on a closed step | `with step.substep(...) as s: pass` then `s.substep(...)`                                 | Warns: the new step attaches to the stack top, which is no longer `s`                                                         |
 
 ## Out of scope
 

python/lib/sift_client/_tests/pytest_plugin/test_pass_fail.py

@@ -17,10 +17,12 @@
 from __future__ import annotations
 
 import textwrap
+import warnings
 
 import pytest
 
 from sift_client._tests.pytest_plugin import _step_status_capture as capture
+from sift_client.errors import SiftReportLifecycleWarning
 from sift_client.sift_types.test_report import TestStatus
 
 pytest_plugins = ["pytester"]
@@ -731,7 +733,7 @@ def test_flash(self, step):
     assert substep.statuses[-1] == TestStatus.FAILED
     assert klass.statuses[-1] == TestStatus.FAILED
     # The module step is the shallowest; it inherits the failure too.
-    module = min(capture._steps().values(), key=lambda s: s.step_path.count("."))
+    module = capture.module_step()
     assert module.statuses[-1] == TestStatus.FAILED
 
 
@@ -761,7 +763,7 @@ def test_flash(self, step):
     assert leaf.statuses[-1] == TestStatus.ABORTED
     assert substep.statuses[-1] == TestStatus.FAILED
     assert klass.statuses[-1] == TestStatus.ABORTED
-    module = min(capture._steps().values(), key=lambda s: s.step_path.count("."))
+    module = capture.module_step()
     assert module.statuses[-1] == TestStatus.ABORTED
 
 
@@ -787,5 +789,127 @@ def test_flash(self, step):
     assert klass is not None
     assert leaf.statuses[-1] == TestStatus.ABORTED
     assert klass.statuses[-1] == TestStatus.ABORTED
-    module = min(capture._steps().values(), key=lambda s: s.step_path.count("."))
+    module = capture.module_step()
     assert module.statuses[-1] == TestStatus.ABORTED
+
+
+# ---------------------------------------------------------------------------
+# Steps used after their ``with`` block exited
+# ---------------------------------------------------------------------------
+
+
+def test_late_failure_corrects_closed_step_and_ancestors(inner):
+    # Case: API-10
+    _run(
+        inner,
+        """
+        def test_x(step):
+            with step.substep(name="inner") as s:
+                pass
+            s.measure(name="m", value=10.0, bounds={"min": 0.0, "max": 5.0})
+        """,
+    )
+    # The substep already resolved PASSED on close, so nothing was left to read
+    # the failure flag; it is corrected in place instead. The chain above it is
+    # still open at that point and resolves FAILED through the normal path, so
+    # the roll-up reaches the module step rather than only the report.
+    substep = next(iter(capture.steps_by_name("inner")), None)
+    assert substep is not None
+    assert substep.statuses[-2:] == [TestStatus.PASSED, TestStatus.FAILED]
+    assert capture.final_status("test_x") == TestStatus.FAILED
+    module = capture.module_step()
+    assert module.statuses[-1] == TestStatus.FAILED
+
+
+def test_late_passing_measurement_leaves_statuses_alone(inner):
+    # Case: API-11
+    _run(
+        inner,
+        """
+        def test_x(step):
+            with step.substep(name="inner") as s:
+                pass
+            s.measure(name="m", value=1.0, bounds={"min": 0.0, "max": 5.0})
+        """,
+    )
+    # The same mis-ordering with an in-bounds value: the measurement attaches to
+    # the right step and there is nothing to correct. Only the warning separates
+    # this from API-10, which is why it fires on the ordering and not the outcome.
+    substep = next(iter(capture.steps_by_name("inner")), None)
+    assert substep is not None
+    assert substep.statuses[-1] == TestStatus.PASSED
+    assert capture.final_status("test_x") == TestStatus.PASSED
+
+
+@pytest.fixture
+def simulate_context():
+    """A real ``ReportContext`` on the ``--sift-disabled`` simulate path.
+
+    Exercises the actual ``TestStep`` objects and statuses without contacting Sift.
+    """
+    from sift_client._internal.pytest_plugin.report import build_disabled_client
+    from sift_client.util.test_results import ReportContext
+
+    return ReportContext(build_disabled_client(), name="n", log_file=False)
+
+
+@pytest.fixture
+def closed_step(simulate_context):
+    """A step whose ``with`` block has already exited."""
+    step = simulate_context.new_step(name="s")
+    with step:
+        pass
+    return step
+
+
+class TestClosedStepWarning:
+    """Using a step after its ``with`` block exited warns, whatever the outcome."""
+
+    def test_warning_is_latched_per_operation(self, closed_step) -> None:
+        with pytest.warns(SiftReportLifecycleWarning, match=r"a result was recorded on step 's'"):
+            closed_step.measure(name="a", value=1.0, bounds={"min": 0.0, "max": 5.0})
+        # A repeat is latched: a wrapper doing this on every step would otherwise
+        # flood the runner's warning summary. The per-call record is in the audit
+        # log instead.
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", SiftReportLifecycleWarning)
+            closed_step.measure(name="b", value=1.0, bounds={"min": 0.0, "max": 5.0})
+        # A substep records no outcome, so it never reaches the chokepoint the
+        # measure family shares, and it fails differently (it mis-parents as well
+        # as losing the status). Its own guard, its own signal.
+        with pytest.warns(SiftReportLifecycleWarning, match="a substep was recorded"):
+            closed_step.substep(name="late")
+
+    def test_measure_all_warns_even_when_it_records_nothing(self, closed_step) -> None:
+        # An all-in-bounds series records no measurement, so the delegated
+        # ``measure`` never runs; the trailing ``record_step_outcome`` still does.
+        with pytest.warns(SiftReportLifecycleWarning, match="a result was recorded"):
+            closed_step.measure_all(name="m", values=[1.0, 2.0], bounds={"min": 0.0, "max": 5.0})
+
+    def test_open_step_does_not_warn(self, simulate_context) -> None:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", SiftReportLifecycleWarning)
+            with simulate_context.new_step(name="s") as step:
+                step.measure(name="m", value=1.0, bounds={"min": 0.0, "max": 5.0})
+                step.measure_all(name="series", values=[1.0], bounds={"min": 0.0, "max": 5.0})
+                step.measure_avg(name="avg", values=[1.0], bounds={"min": 0.0, "max": 5.0})
+                step.report_outcome(name="check", result=True)
+                with step.substep(name="inner"):
+                    pass
+
+    def test_late_failure_after_finalize_warns_and_writes_nothing(
+        self, simulate_context, closed_step
+    ) -> None:
+        current = closed_step.current_step
+        assert current is not None
+        simulate_context.finalize()
+        with pytest.warns(SiftReportLifecycleWarning) as recorded:
+            closed_step.measure(name="m", value=10.0, bounds={"min": 0.0, "max": 5.0})
+        # Both signals fire: the call is mis-ordered, and the correction that would
+        # normally absorb that has nowhere left to go.
+        messages = [str(w.message) for w in recorded]
+        assert any("a result was recorded" in m for m in messages)
+        assert any("after the report was finalized" in m for m in messages)
+        # Past finalize the report status is already written, so the step keeps what
+        # it closed with.
+        assert current.status == TestStatus.PASSED

python/lib/sift_client/errors.py

@@ -19,6 +19,10 @@ class SiftIgnoredInputWarning(SiftWarning):
     """
 
 
+class SiftReportLifecycleWarning(SiftWarning):
+    """Warning for test-results updates that are out of the intended lifecycle order."""
+
+
 class SiftCredentialsError(ValueError):
     """Raised when Sift credentials cannot be resolved.
 

python/lib/sift_client/util/test_results/__init__.py

@@ -25,6 +25,9 @@
             return result # This is optional for other uses, but the step and its parents will be updated correctly i.e. failed if the measurement fails.
 ```
 
+Measure inside the step's `with` block. Closing a step resolves its status and
+rolls the result up to its parents.
+
 #### Manually Updating Underlyling Report
 You can also manually update the underlying report or steps by accessing the context manager's attributes.
 ```python