diff --git a/crates/libs/api/src/controllers/sandbox/responses.rs b/crates/libs/api/src/controllers/sandbox/responses.rs index cb11644058..4e98d79b41 100644 --- a/crates/libs/api/src/controllers/sandbox/responses.rs +++ b/crates/libs/api/src/controllers/sandbox/responses.rs @@ -569,9 +569,11 @@ pub(super) fn sandbox_to_cached_response( ) -> SandboxResponse { let mut response = sandbox_to_response(sandbox, None); if let Some(observed) = observed { - response.set_state(SandboxStateInput::new( - Some(sandbox.lifecycle.clone()), + response.set_state(SandboxStateInput::observed_on( + sandbox.lifecycle.clone(), + sandbox.active_node_id, observed.status.clone(), + observed.node_id, )); response.spaced_version = observed.spaced_version.clone(); response.pid = observed.pid.and_then(|value| u32::try_from(value).ok()); diff --git a/crates/libs/api/src/models/sandbox_state.rs b/crates/libs/api/src/models/sandbox_state.rs index 20b3d5e542..8e739aa0c1 100644 --- a/crates/libs/api/src/models/sandbox_state.rs +++ b/crates/libs/api/src/models/sandbox_state.rs @@ -1,3 +1,5 @@ +use pplx_space_common_core::types::{NodeId, SandboxLifecycle}; + /// What the admin response's `lifecycle` and `status` and the ECW sandbox /// state are rendered from: the row's stored lifecycle and the status its node /// last observed, kept apart so each surface decides how the two combine. @@ -11,16 +13,50 @@ pub struct SandboxStateInput { stored_lifecycle: Option, observed_status: Option, + observation: Observation, +} + +/// Whether the observed status describes the row's current execution. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum Observation { + #[default] + CurrentExecution, + /// Published by a node other than the row's active node, such as the + /// node the sandbox left or the one it was lost on. + OtherNode, } impl SandboxStateInput { /// `stored_lifecycle` is absent for a sandbox only a node reports; /// `observed_status` is absent without an observed record or when the - /// record carries no status. + /// record carries no status. The observation is taken to describe the + /// row's current execution. pub fn new(stored_lifecycle: Option, observed_status: Option) -> Self { Self { stored_lifecycle, observed_status, + observation: Observation::CurrentExecution, + } + } + + /// A row read with the status `observed_node` last published for it. The + /// observation describes the row's current execution only when that node + /// is the row's active node; the record carries no execution generation + /// to compare. + pub fn observed_on( + stored_lifecycle: String, + active_node: Option, + observed_status: Option, + observed_node: Option, + ) -> Self { + let observation = match (active_node, observed_node) { + (Some(active), Some(observed)) if active == observed => Observation::CurrentExecution, + _ => Observation::OtherNode, + }; + Self { + stored_lifecycle: Some(stored_lifecycle), + observed_status, + observation, } } @@ -45,4 +81,63 @@ impl SandboxStateInput { .or(self.stored_lifecycle()) .map(str::to_string) } + + /// Whether the row is a delete tombstone, `deleted` or `cleanup_required`: + /// terminal, and outside `SandboxLifecycle`. + pub fn is_tombstone(&self) -> bool { + self.stored_lifecycle().is_some_and(is_tombstone) + } + + /// The lifecycle a caller-facing state is rendered from. The stored + /// lifecycle decides every terminal and at-rest state; the observed + /// status refines only a live one, and only when it describes the row's + /// current execution. + pub fn rendered_lifecycle(&self) -> Option<&str> { + let stored = self.stored_lifecycle(); + if stored.is_some_and(stored_lifecycle_decides) { + return stored; + } + match self.observation { + Observation::CurrentExecution => self.observed_status().or(stored), + Observation::OtherNode => stored, + } + } +} + +/// Whether a stored lifecycle is one the node's observation may not override: +/// the control plane put the row there, and a node still reporting the +/// execution it had is behind it. +fn stored_lifecycle_decides(stored: &str) -> bool { + let normalized = stored.trim().to_ascii_lowercase(); + match normalized.parse::() { + Ok( + SandboxLifecycle::Failed + | SandboxLifecycle::Quarantined + | SandboxLifecycle::Stopped + | SandboxLifecycle::Suspended + | SandboxLifecycle::Deleting, + ) => true, + Ok( + SandboxLifecycle::Created + | SandboxLifecycle::Creating + | SandboxLifecycle::Starting + | SandboxLifecycle::Started + | SandboxLifecycle::Restoring + | SandboxLifecycle::Pausing + | SandboxLifecycle::Paused + | SandboxLifecycle::Resuming + | SandboxLifecycle::Suspending + | SandboxLifecycle::Stopping + | SandboxLifecycle::Checkpointing, + ) => false, + // An unknown value keeps reading the node. + Err(_) => is_tombstone(&normalized), + } +} + +fn is_tombstone(stored: &str) -> bool { + matches!( + stored.trim().to_ascii_lowercase().as_str(), + "deleted" | "cleanup_required" + ) } diff --git a/crates/libs/api/src/routes/e2b/snapshots/pplx_space_api__routes__e2b__state_render_tests__each_surface_renders_stored_lifecycle_and_observed_status.snap b/crates/libs/api/src/routes/e2b/snapshots/pplx_space_api__routes__e2b__state_render_tests__each_surface_renders_stored_lifecycle_and_observed_status.snap index f4151b1f6e..b4dc5ef25a 100644 --- a/crates/libs/api/src/routes/e2b/snapshots/pplx_space_api__routes__e2b__state_render_tests__each_surface_renders_stored_lifecycle_and_observed_status.snap +++ b/crates/libs/api/src/routes/e2b/snapshots/pplx_space_api__routes__e2b__state_render_tests__each_surface_renders_stored_lifecycle_and_observed_status.snap @@ -30,6 +30,9 @@ stored=created observed="FAILED" admin="FAILED"/"FAILED" stored=created observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=created observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=created observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=created observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=created observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=created observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=creating observed=absent admin="creating"/"creating" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=creating observed=null admin="creating"/"creating" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=creating observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false @@ -58,6 +61,9 @@ stored=creating observed="FAILED" admin="FAILED"/"FAILED" stored=creating observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=creating observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=creating observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=creating observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=creating observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false +stored=creating observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=creating ready=false stored=starting observed=absent admin="starting"/"starting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false stored=starting observed=null admin="starting"/"starting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false stored=starting observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false @@ -86,6 +92,9 @@ stored=starting observed="FAILED" admin="FAILED"/"FAILED" stored=starting observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false stored=starting observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false stored=starting observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false +stored=starting observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false +stored=starting observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false +stored=starting observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=starting ready=false stored=started observed=absent admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true stored=started observed=null admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true stored=started observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true @@ -114,6 +123,9 @@ stored=started observed="FAILED" admin="FAILED"/"FAILED" stored=started observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true stored=started observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true stored=started observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true +stored=started observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true +stored=started observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true +stored=started observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=running ready=true stored=restoring observed=absent admin="restoring"/"restoring" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false stored=restoring observed=null admin="restoring"/"restoring" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false stored=restoring observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false @@ -142,6 +154,9 @@ stored=restoring observed="FAILED" admin="FAILED"/"FAILED" stored=restoring observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false stored=restoring observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false stored=restoring observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false +stored=restoring observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false +stored=restoring observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false +stored=restoring observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=restoring ready=false stored=pausing observed=absent admin="pausing"/"pausing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false stored=pausing observed=null admin="pausing"/"pausing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false stored=pausing observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false @@ -170,6 +185,9 @@ stored=pausing observed="FAILED" admin="FAILED"/"FAILED" stored=pausing observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false stored=pausing observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false stored=pausing observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false +stored=pausing observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false +stored=pausing observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false +stored=pausing observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=pausing ready=false stored=paused observed=absent admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=paused ready=false stored=paused observed=null admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=paused ready=false stored=paused observed="unspecified" admin="unspecified"/"unspecified" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=paused ready=false @@ -198,6 +216,9 @@ stored=paused observed="FAILED" admin="FAILED"/"FAILED" stored=paused observed="lost" admin="lost"/"lost" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=paused ready=false stored=paused observed="" admin=""/"" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=paused ready=false stored=paused observed=" " admin=" "/" " ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=paused ready=false +stored=paused observed=other "started" admin="started"/"started" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=paused ready=false +stored=paused observed=other "paused" admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=paused ready=false +stored=paused observed=other "failed" admin="failed"/"failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=paused ready=false stored=resuming observed=absent admin="resuming"/"resuming" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false stored=resuming observed=null admin="resuming"/"resuming" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false stored=resuming observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false @@ -226,6 +247,9 @@ stored=resuming observed="FAILED" admin="FAILED"/"FAILED" stored=resuming observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false stored=resuming observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false stored=resuming observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false +stored=resuming observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false +stored=resuming observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false +stored=resuming observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=resuming ready=false stored=suspending observed=absent admin="suspending"/"suspending" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=suspending observed=null admin="suspending"/"suspending" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=suspending observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false @@ -254,34 +278,40 @@ stored=suspending observed="FAILED" admin="FAILED"/"FAILED" stored=suspending observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=suspending observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=suspending observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=suspending observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=suspending observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=suspending observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=suspended observed=absent admin="suspended"/"suspended" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=suspended observed=null admin="suspended"/"suspended" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="unspecified" admin="unspecified"/"unspecified" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="creating" admin="creating"/"creating" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="created" admin="created"/"created" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="starting" admin="starting"/"starting" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="started" admin="started"/"started" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="stopping" admin="stopping"/"stopping" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="stopped" admin="stopped"/"stopped" ecw list=paused get=404 gate=404 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="deleting" admin="deleting"/"deleting" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="pausing" admin="pausing"/"pausing" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="unspecified" admin="unspecified"/"unspecified" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="creating" admin="creating"/"creating" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="created" admin="created"/"created" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="starting" admin="starting"/"starting" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="started" admin="started"/"started" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="stopping" admin="stopping"/"stopping" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="stopped" admin="stopped"/"stopped" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="deleting" admin="deleting"/"deleting" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="pausing" admin="pausing"/"pausing" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=suspended observed="paused" admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="resuming" admin="resuming"/"resuming" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="suspending" admin="suspending"/"suspending" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="resuming" admin="resuming"/"resuming" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="suspending" admin="suspending"/"suspending" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=suspended observed="suspended" admin="suspended"/"suspended" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="restoring" admin="restoring"/"restoring" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="quarantined" admin="quarantined"/"quarantined" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="failed" admin="failed"/"failed" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="restoring" admin="restoring"/"restoring" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="quarantined" admin="quarantined"/"quarantined" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="failed" admin="failed"/"failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=suspended observed=" Paused " admin=" Paused "/" Paused " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=suspended observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="Stopped" admin="Stopped"/"Stopped" ecw list=paused get=404 gate=404 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="FAILED" admin="FAILED"/"FAILED" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="lost" admin="lost"/"lost" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed="" admin=""/"" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false -stored=suspended observed=" " admin=" "/" " ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="Stopped" admin="Stopped"/"Stopped" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="FAILED" admin="FAILED"/"FAILED" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="lost" admin="lost"/"lost" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed="" admin=""/"" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed=" " admin=" "/" " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed=other "started" admin="started"/"started" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed=other "paused" admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false +stored=suspended observed=other "failed" admin="failed"/"failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=suspended ready=false stored=stopping observed=absent admin="stopping"/"stopping" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false stored=stopping observed=null admin="stopping"/"stopping" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false stored=stopping observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false @@ -310,34 +340,40 @@ stored=stopping observed="FAILED" admin="FAILED"/"FAILED" stored=stopping observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false stored=stopping observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false stored=stopping observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false +stored=stopping observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false +stored=stopping observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false +stored=stopping observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopping ready=false stored=stopped observed=absent admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false stored=stopped observed=null admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="creating" admin="creating"/"creating" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="created" admin="created"/"created" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="starting" admin="starting"/"starting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="started" admin="started"/"started" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false +stored=stopped observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="creating" admin="creating"/"creating" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="created" admin="created"/"created" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="starting" admin="starting"/"starting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false stored=stopped observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="paused" admin="paused"/"paused" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false stored=stopped observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false -stored=stopped observed="lost" admin="lost"/"lost" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed="" admin=""/"" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false -stored=stopped observed=" " admin=" "/" " ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=stopped ready=false +stored=stopped observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="lost" admin="lost"/"lost" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed="" admin=""/"" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed=" " admin=" "/" " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed=other "started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed=other "paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false +stored=stopped observed=other "failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=stopped ready=false stored=deleting observed=absent admin="deleting"/"deleting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed=null admin="deleting"/"deleting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false @@ -346,26 +382,29 @@ stored=deleting observed="created" admin="created"/"created" stored=deleting observed="starting" admin="starting"/"starting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="stopping" admin="stopping"/"stopping" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false -stored=deleting observed="stopped" admin="stopped"/"stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false +stored=deleting observed="stopped" admin="stopped"/"stopped" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="deleting" admin="deleting"/"deleting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="pausing" admin="pausing"/"pausing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false -stored=deleting observed="paused" admin="paused"/"paused" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false +stored=deleting observed="paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="resuming" admin="resuming"/"resuming" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="suspending" admin="suspending"/"suspending" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false -stored=deleting observed="suspended" admin="suspended"/"suspended" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false +stored=deleting observed="suspended" admin="suspended"/"suspended" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="restoring" admin="restoring"/"restoring" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false -stored=deleting observed="quarantined" admin="quarantined"/"quarantined" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false -stored=deleting observed="failed" admin="failed"/"failed" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false -stored=deleting observed=" Paused " admin=" Paused "/" Paused " ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false -stored=deleting observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false -stored=deleting observed="Stopped" admin="Stopped"/"Stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false -stored=deleting observed="FAILED" admin="FAILED"/"FAILED" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=deleting ready=false +stored=deleting observed="quarantined" admin="quarantined"/"quarantined" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed="failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed=" Paused " admin=" Paused "/" Paused " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed="Stopped" admin="Stopped"/"Stopped" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed="FAILED" admin="FAILED"/"FAILED" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=deleting observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false +stored=deleting observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=deleting ready=false stored=checkpointing observed=absent admin="checkpointing"/"checkpointing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=checkpointing observed=null admin="checkpointing"/"checkpointing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=checkpointing observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false @@ -394,118 +433,133 @@ stored=checkpointing observed="FAILED" admin="FAILED"/"FAILED" stored=checkpointing observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=checkpointing observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=checkpointing observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=checkpointing observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=checkpointing observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false +stored=checkpointing observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=suspending ready=false stored=quarantined observed=absent admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=quarantined observed=null admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="creating" admin="creating"/"creating" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="created" admin="created"/"created" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="starting" admin="starting"/"starting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="started" admin="started"/"started" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="paused" admin="paused"/"paused" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false +stored=quarantined observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="creating" admin="creating"/"creating" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="created" admin="created"/"created" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="starting" admin="starting"/"starting" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="started" admin="started"/"started" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="paused" admin="paused"/"paused" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=quarantined observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=quarantined observed="failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=quarantined observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=quarantined observed="lost" admin="lost"/"lost" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed="" admin=""/"" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=quarantined observed=" " admin=" "/" " ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false +stored=quarantined observed="lost" admin="lost"/"lost" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed="" admin=""/"" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed=" " admin=" "/" " ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed=other "started" admin="started"/"started" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed=other "paused" admin="paused"/"paused" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=quarantined observed=other "failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=failed observed=absent admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=failed observed=null admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="creating" admin="creating"/"creating" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="created" admin="created"/"created" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="starting" admin="starting"/"starting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="started" admin="started"/"started" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="paused" admin="paused"/"paused" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false +stored=failed observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="creating" admin="creating"/"creating" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="created" admin="created"/"created" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="starting" admin="starting"/"starting" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="started" admin="started"/"started" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="paused" admin="paused"/"paused" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=failed observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=failed observed="failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false stored=failed observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false -stored=failed observed="lost" admin="lost"/"lost" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed="" admin=""/"" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=failed observed=" " admin=" "/" " ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=failed ready=false -stored=deleted observed=absent admin="deleted"/"deleted" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed=null admin="deleted"/"deleted" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="creating" admin="creating"/"creating" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="created" admin="created"/"created" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="starting" admin="starting"/"starting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="stopping" admin="stopping"/"stopping" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="stopped" admin="stopped"/"stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="deleting" admin="deleting"/"deleting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="pausing" admin="pausing"/"pausing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="paused" admin="paused"/"paused" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="resuming" admin="resuming"/"resuming" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="suspending" admin="suspending"/"suspending" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="suspended" admin="suspended"/"suspended" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="restoring" admin="restoring"/"restoring" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="quarantined" admin="quarantined"/"quarantined" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="failed" admin="failed"/"failed" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed=" Paused " admin=" Paused "/" Paused " ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="Stopped" admin="Stopped"/"Stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="FAILED" admin="FAILED"/"FAILED" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 -stored=deleted observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=deleted observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=404 -stored=cleanup_required observed=absent admin="cleanup_required"/"cleanup_required" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed=null admin="cleanup_required"/"cleanup_required" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="creating" admin="creating"/"creating" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="created" admin="created"/"created" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="starting" admin="starting"/"starting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="stopping" admin="stopping"/"stopping" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="stopped" admin="stopped"/"stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="deleting" admin="deleting"/"deleting" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="pausing" admin="pausing"/"pausing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="paused" admin="paused"/"paused" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="resuming" admin="resuming"/"resuming" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="suspending" admin="suspending"/"suspending" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="suspended" admin="suspended"/"suspended" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="restoring" admin="restoring"/"restoring" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="quarantined" admin="quarantined"/"quarantined" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="failed" admin="failed"/"failed" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed=" Paused " admin=" Paused "/" Paused " ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=running get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="Stopped" admin="Stopped"/"Stopped" ecw list=running get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="FAILED" admin="FAILED"/"FAILED" ecw list=running get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored=cleanup_required observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored=cleanup_required observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 +stored=failed observed="lost" admin="lost"/"lost" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed="" admin=""/"" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed=" " admin=" "/" " ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed=other "started" admin="started"/"started" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed=other "paused" admin="paused"/"paused" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=failed observed=other "failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=failed ready=false +stored=deleted observed=absent admin="deleted"/"deleted" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=null admin="deleted"/"deleted" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="creating" admin="creating"/"creating" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="created" admin="created"/"created" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="starting" admin="starting"/"starting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="lost" admin="lost"/"lost" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed="" admin=""/"" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=" " admin=" "/" " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=other "started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=other "paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=deleted observed=other "failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=absent admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=null admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="creating" admin="creating"/"creating" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="created" admin="created"/"created" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="starting" admin="starting"/"starting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="lost" admin="lost"/"lost" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed="" admin=""/"" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=" " admin=" "/" " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=other "started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=other "paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 +stored=cleanup_required observed=other "failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=404 stored=bogus observed=absent admin="bogus"/"bogus" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 stored=bogus observed=null admin="bogus"/"bogus" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 stored=bogus observed="unspecified" admin="unspecified"/"unspecified" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 @@ -534,59 +588,68 @@ stored=bogus observed="FAILED" admin="FAILED"/"FAILED" stored=bogus observed="lost" admin="lost"/"lost" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 stored=bogus observed="" admin=""/"" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 stored=bogus observed=" " admin=" "/" " ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 +stored=bogus observed=other "started" admin="started"/"started" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 +stored=bogus observed=other "paused" admin="paused"/"paused" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 +stored=bogus observed=other "failed" admin="failed"/"failed" ecw list=running get=running gate=pass connect wake=false pod=true not_started=Wake unreachable=Retry spent=Wake native=500 stored= Suspended observed=absent admin=" Suspended "/" Suspended " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Suspended observed=null admin=" Suspended "/" Suspended " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="unspecified" admin="unspecified"/"unspecified" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="creating" admin="creating"/"creating" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="created" admin="created"/"created" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="starting" admin="starting"/"starting" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="started" admin="started"/"started" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="stopping" admin="stopping"/"stopping" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="stopped" admin="stopped"/"stopped" ecw list=paused get=404 gate=404 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="deleting" admin="deleting"/"deleting" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="pausing" admin="pausing"/"pausing" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="unspecified" admin="unspecified"/"unspecified" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="creating" admin="creating"/"creating" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="created" admin="created"/"created" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="starting" admin="starting"/"starting" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="started" admin="started"/"started" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="stopping" admin="stopping"/"stopping" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="stopped" admin="stopped"/"stopped" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="deleting" admin="deleting"/"deleting" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="pausing" admin="pausing"/"pausing" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Suspended observed="paused" admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="resuming" admin="resuming"/"resuming" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="suspending" admin="suspending"/"suspending" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="resuming" admin="resuming"/"resuming" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="suspending" admin="suspending"/"suspending" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Suspended observed="suspended" admin="suspended"/"suspended" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="restoring" admin="restoring"/"restoring" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="quarantined" admin="quarantined"/"quarantined" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="failed" admin="failed"/"failed" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="restoring" admin="restoring"/"restoring" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="quarantined" admin="quarantined"/"quarantined" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="failed" admin="failed"/"failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Suspended observed=" Paused " admin=" Paused "/" Paused " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Suspended observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="Stopped" admin="Stopped"/"Stopped" ecw list=paused get=404 gate=404 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="FAILED" admin="FAILED"/"FAILED" ecw list=paused get=failed gate=500 connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="lost" admin="lost"/"lost" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed="" admin=""/"" ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 -stored= Suspended observed=" " admin=" "/" " ecw list=paused get=running gate=pass connect wake=true pod=true not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="Stopped" admin="Stopped"/"Stopped" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="FAILED" admin="FAILED"/"FAILED" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="lost" admin="lost"/"lost" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed="" admin=""/"" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed=" " admin=" "/" " ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed=other "started" admin="started"/"started" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed=other "paused" admin="paused"/"paused" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 +stored= Suspended observed=other "failed" admin="failed"/"failed" ecw list=paused get=paused gate=pass connect wake=true pod=false not_started=Wake unreachable=Wake spent=Wake native=500 stored= Stopped observed=absent admin=" Stopped "/" Stopped " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 stored= Stopped observed=null admin=" Stopped "/" Stopped " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="creating" admin="creating"/"creating" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="created" admin="created"/"created" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="starting" admin="starting"/"starting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="started" admin="started"/"started" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 +stored= Stopped observed="unspecified" admin="unspecified"/"unspecified" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="creating" admin="creating"/"creating" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="created" admin="created"/"created" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="starting" admin="starting"/"starting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="stopping" admin="stopping"/"stopping" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 stored= Stopped observed="stopped" admin="stopped"/"stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="paused" admin="paused"/"paused" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="failed" admin="failed"/"failed" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=paused gate=pass connect wake=true pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="deleting" admin="deleting"/"deleting" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="pausing" admin="pausing"/"pausing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="resuming" admin="resuming"/"resuming" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="checkpointing" admin="checkpointing"/"checkpointing" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="suspending" admin="suspending"/"suspending" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="suspended" admin="suspended"/"suspended" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="restoring" admin="restoring"/"restoring" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="cleanup_required" admin="cleanup_required"/"cleanup_required" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="cleanup_failed" admin="cleanup_failed"/"cleanup_failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="quarantined" admin="quarantined"/"quarantined" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed=" Paused " admin=" Paused "/" Paused " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="SUSPENDED" admin="SUSPENDED"/"SUSPENDED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 stored= Stopped observed="Stopped" admin="Stopped"/"Stopped" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=failed gate=500 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 -stored= Stopped observed="lost" admin="lost"/"lost" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed="" admin=""/"" ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 -stored= Stopped observed=" " admin=" "/" " ecw list=hidden get=running gate=pass connect wake=false pod=true not_started=Timeout unreachable=Retry spent=Timeout native=500 +stored= Stopped observed="FAILED" admin="FAILED"/"FAILED" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="lost" admin="lost"/"lost" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed="" admin=""/"" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed=" " admin=" "/" " ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed=other "started" admin="started"/"started" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed=other "paused" admin="paused"/"paused" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 +stored= Stopped observed=other "failed" admin="failed"/"failed" ecw list=hidden get=404 gate=404 connect wake=false pod=false not_started=Timeout unreachable=Timeout spent=Timeout native=500 diff --git a/crates/libs/api/src/routes/e2b/state_render_tests.rs b/crates/libs/api/src/routes/e2b/state_render_tests.rs index 527e17bb5f..4e8f685401 100644 --- a/crates/libs/api/src/routes/e2b/state_render_tests.rs +++ b/crates/libs/api/src/routes/e2b/state_render_tests.rs @@ -2,7 +2,8 @@ //! lifecycle and every status a node can publish for it. //! //! Each row inserts a sandbox row and, unless the observation is absent, its -//! `sandbox_observed_states` record, then reads it the way each surface does: +//! `sandbox_observed_states` record, published by the row's active node or by +//! another node, then reads it the way each surface does: //! the admin response and the ECW get, connect, timeout and refresh paths //! through `tracked_response`, the ECW list through `cached_sandbox_response`, //! and the native `GET /v1/sandboxes/{id}` through `read_public_sandbox`. The @@ -69,6 +70,10 @@ const ODD_STATUSES: &[&str] = &[ " ", ]; +/// Statuses a node other than the row's active node still reports, such as +/// the node a sandbox left or was lost on. +const OTHER_NODE_STATUSES: &[&str] = &["started", "paused", "failed"]; + #[derive(Clone, Copy)] enum Observed { /// No `sandbox_observed_states` record. @@ -76,6 +81,8 @@ enum Observed { /// A record whose `status` is NULL. NullStatus, Status(&'static str), + /// A record published by a node other than the row's active node. + OtherNode(&'static str), } impl Observed { @@ -84,6 +91,7 @@ impl Observed { Self::Absent => "absent".to_string(), Self::NullStatus => "null".to_string(), Self::Status(status) => format!("{status:?}"), + Self::OtherNode(status) => format!("other {status:?}"), } } } @@ -102,6 +110,7 @@ fn observations() -> Vec { .into_iter() .chain(NODE_STATUSES.iter().copied().map(Observed::Status)) .chain(ODD_STATUSES.iter().copied().map(Observed::Status)) + .chain(OTHER_NODE_STATUSES.iter().copied().map(Observed::OtherNode)) .collect() } @@ -130,7 +139,7 @@ async fn insert_node(pool: &PgPool, node_id: NodeId) -> TestResult { async fn insert_row( pool: &PgPool, sandbox_id: SandboxId, - node_id: NodeId, + [node_id, other_node_id]: [NodeId; 2], stored: &str, observed: Observed, ) -> TestResult { @@ -148,17 +157,18 @@ async fn insert_row( .bind(node_id) .execute(pool) .await?; - let status = match observed { + let (observed_node_id, status) = match observed { Observed::Absent => return Ok(()), - Observed::NullStatus => None, - Observed::Status(status) => Some(status), + Observed::NullStatus => (node_id, None), + Observed::Status(status) => (node_id, Some(status)), + Observed::OtherNode(status) => (other_node_id, Some(status)), }; sqlx::query( "INSERT INTO sandbox_observed_states (sandbox_id, node_id, status, template) VALUES ($1, $2, $3, 'seed')", ) .bind(sandbox_id) - .bind(node_id) + .bind(observed_node_id) .bind(status) .execute(pool) .await?; @@ -246,14 +256,16 @@ async fn each_surface_renders_stored_lifecycle_and_observed_status() -> TestResu let db = pg_test_db().await?; let state = AppState::from_pool(db.pool.clone()).await?; let keyring = EcwCredentialKeyring::from_current_key_for_test(b"state-render-table-key"); - let node_id = NodeId::new(); - insert_node(&db.pool, node_id).await?; + let nodes = [NodeId::new(), NodeId::new()]; + for node_id in nodes { + insert_node(&db.pool, node_id).await?; + } let mut table = String::new(); for stored in stored_lifecycles() { for observed in observations() { let sandbox_id = SandboxId::new(); - insert_row(&db.pool, sandbox_id, node_id, &stored, observed).await?; + insert_row(&db.pool, sandbox_id, nodes, &stored, observed).await?; let rendered = render_row(&state, &keyring, sandbox_id).await?; writeln!( table, diff --git a/crates/libs/api/src/routes/e2b/tests.rs b/crates/libs/api/src/routes/e2b/tests.rs index 75dfc239be..fb1eee8217 100644 --- a/crates/libs/api/src/routes/e2b/tests.rs +++ b/crates/libs/api/src/routes/e2b/tests.rs @@ -3997,7 +3997,7 @@ async fn connect_route_never_bad_gateways_for_db_state_shapes() -> TestResult { "cleanup_required", "unplaced", ConnectRouteActiveNode::None, - StatusCode::CONFLICT, + StatusCode::NOT_FOUND, ), ConnectRouteDbShape::error( "deleted teardown row", diff --git a/crates/libs/api/src/routes/e2b/translate.rs b/crates/libs/api/src/routes/e2b/translate.rs index f282a677a0..5b17b3cf8e 100644 --- a/crates/libs/api/src/routes/e2b/translate.rs +++ b/crates/libs/api/src/routes/e2b/translate.rs @@ -946,13 +946,13 @@ fn terminal_sandbox_state(native: &SandboxResponse) -> Option String { - // The node's observed status wins over the stored lifecycle; a sandbox - // with neither reads as started. + // A sandbox with neither a stored lifecycle nor an observed status reads + // as started. state - .observed_status() - .or(state.stored_lifecycle()) + .rendered_lifecycle() .unwrap_or("started") .trim() .to_ascii_lowercase() @@ -963,9 +963,12 @@ pub(super) fn translate_lifecycle_state(lifecycle: &str) -> Option Option { + if state.is_tombstone() { + return None; + } match ecw_lifecycle(state).as_str() { "paused" | "suspended" => Some(SandboxStateE2b::Paused), "stopped" | "failed" | "quarantined" => None, @@ -973,9 +976,12 @@ fn ecw_sandbox_state(state: &SandboxStateInput) -> Option { } } -/// Whether ECW reports a terminal sandbox as failed, or as stopped and so -/// not found. +/// Whether ECW reports a terminal sandbox as failed, or as stopped or +/// deleted and so not found. fn ecw_terminal_state(state: &SandboxStateInput) -> Option { + if state.is_tombstone() { + return Some(TerminalSandboxState::Stopped); + } match ecw_lifecycle(state).as_str() { "failed" | "quarantined" => Some(TerminalSandboxState::Failed), "stopped" => Some(TerminalSandboxState::Stopped), diff --git a/crates/libs/api/src/routes/public/sandboxes.rs b/crates/libs/api/src/routes/public/sandboxes.rs index b8d234d77f..636badc058 100644 --- a/crates/libs/api/src/routes/public/sandboxes.rs +++ b/crates/libs/api/src/routes/public/sandboxes.rs @@ -1300,13 +1300,16 @@ enum PublicSandboxQuery { fn public_sandbox_query(query: PublicSandboxQuery) -> sqlx::AssertSqlSafe { // Control-plane retention keeps deleted tombstones, but deleted public - // resources are NotFound and must not consume a list page's lookahead. + // resources are NotFound and must not consume a list page's lookahead. A + // `cleanup_required` tombstone, a delete whose cleanup is still owed, + // reads as deleted too. let sql = match query { PublicSandboxQuery::ById => format!( "{PUBLIC_SANDBOX_SELECT}, FALSE AS has_next FROM sandboxes s {PUBLIC_SANDBOX_JOINS} - WHERE s.id = $8 AND s.org_id = $9 AND s.lifecycle <> 'deleted'" + WHERE s.id = $8 AND s.org_id = $9 + AND s.lifecycle NOT IN ('deleted', 'cleanup_required')" ), PublicSandboxQuery::Page => format!( "{page_rows} diff --git a/crates/libs/api/src/routes/public/sandboxes/tests.rs b/crates/libs/api/src/routes/public/sandboxes/tests.rs index 8860471b9b..f66015c440 100644 --- a/crates/libs/api/src/routes/public/sandboxes/tests.rs +++ b/crates/libs/api/src/routes/public/sandboxes/tests.rs @@ -993,6 +993,72 @@ async fn sandbox_reads_exclude_deleted_tombstones_before_pagination() -> TestDbR Ok(()) } +/// `cleanup_required` is a delete tombstone like `deleted`, so `GET +/// /v1/sandboxes/{id}` answers it with the same status and body. +#[tokio::test] +async fn get_answers_a_cleanup_required_tombstone_as_deleted() -> TestDbResult<()> { + Box::pin(async { + const KEY: &str = "tombstone-read-test-key"; + let db = pg_test_db().await?; + sqlx::query(pplx_space_common_core::sql!( + "INSERT INTO api_keys (id, org_id, name, key_hash, allowed_surfaces) + VALUES ($1, $2, 'tombstone-read-test', $3, ARRAY['native'])" + )) + .bind(Uuid::now_v7()) + .bind(SEED_ORG_ID) + .bind(crate::auth::hash_token(KEY)) + .execute(&db.pool) + .await?; + let state = AppState::from_pool(db.pool.clone()).await?; + let app = axum::Router::new() + .nest("/v1", crate::routes::public::v1_router(state.clone())) + .with_state(state); + + let mut answers = Vec::new(); + for lifecycle in ["deleted", "cleanup_required"] { + let id = SandboxId::new(); + sqlx::query(pplx_space_common_core::sql!( + "INSERT INTO sandboxes ( + id, user_id, org_id, lifecycle, network_policy_override_json, + network_policy_generation + ) VALUES ($1, $2, $3, $4, $5, 1)" + )) + .bind(id) + .bind(SEED_USER_ID) + .bind(SEED_ORG_ID) + .bind(lifecycle) + .bind(SqlJson(&pplx_space_net_common::NetworkPolicyV1::default())) + .execute(&db.pool) + .await?; + let request = + axum::http::Request::get(format!("/v1/sandboxes/{}", PublicSandboxId::from(id))) + .header("authorization", format!("Bearer {KEY}")) + .body(axum::body::Body::empty())?; + let response = tower::ServiceExt::oneshot(app.clone(), request).await?; + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024).await?; + let mut body: serde_json::Value = serde_json::from_slice(&body)?; + // Each request has its own ID. + let request_id = body + .as_object_mut() + .and_then(|body| body.remove("request_id")); + assert!(request_id.is_some(), "{lifecycle}: {body}"); + answers.push((lifecycle, status, body)); + } + + let (_, deleted_status, deleted_body) = &answers[0]; + assert_eq!(*deleted_status, StatusCode::NOT_FOUND, "{deleted_body}"); + let (_, status, body) = &answers[1]; + assert_eq!( + (status, body), + (deleted_status, deleted_body), + "cleanup_required must read as deleted" + ); + Ok(()) + }) + .await +} + #[tokio::test] async fn sandbox_reads_expose_only_the_current_failure_episode() -> TestDbResult<()> { let db = pg_test_db().await?;