diff --git a/AGENTS.md b/AGENTS.md index 012969da9f..0ed5902932 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,7 +153,7 @@ ocsf_emit!(event); - `crate::ocsf_ctx()` returns the process-wide `EventContext`. It is always available (falls back to defaults in tests). - `ocsf_emit!()` is non-blocking and cannot panic. It stores the event in a thread-local and emits via `tracing::info!()`. -- In `openshell-supervisor-network`, emit with `crate::telemetry::egress_emit!` instead, or `crate::telemetry::emit_decision` for the records of one decision, so denied HTTP decisions get a deny span. Clippy rejects `ocsf_emit!` there. +- In `openshell-supervisor-network`, the inspected request scope automatically correlates all OCSF records at emission, including findings, and uses the request span without creating a denial-only span. Emit through `crate::telemetry::egress_emit!` or `crate::telemetry::emit_decision`; Clippy rejects direct `ocsf_emit!` calls there. - The shorthand layer and JSONL layer extract the event from the thread-local. The shorthand format is derived automatically from the builder fields. - For security findings, **dual-emit**: one domain event (e.g., `SshActivityBuilder`) AND one `DetectionFindingBuilder` for the same incident. - Never log secrets, credentials, or query parameters in OCSF messages. The OCSF JSONL file may be shipped to external systems. diff --git a/Cargo.lock b/Cargo.lock index d8c338339d..dfb1a7c140 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5341,6 +5341,7 @@ dependencies = [ "openshell-sandbox-backend", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", + "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", "prost-types", @@ -5365,6 +5366,7 @@ dependencies = [ "tonic", "tower-mcp-types", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", "webpki-roots", diff --git a/crates/openshell-ocsf/src/events/mod.rs b/crates/openshell-ocsf/src/events/mod.rs index 66038cd9cb..ee3cf27b69 100644 --- a/crates/openshell-ocsf/src/events/mod.rs +++ b/crates/openshell-ocsf/src/events/mod.rs @@ -281,6 +281,29 @@ mod tests { assert_eq!(deserialized.class_uid(), 2004); } + #[test] + fn correlation_uid_can_be_set_on_any_event_class() { + let ctx = test_sandbox_context(); + let mut event = AuthenticationBuilder::new( + &ctx, + User::other("user-1", "User"), + Service::new("gateway"), + ) + .status(StatusId::Success) + .build(); + + event.base_mut().metadata.correlation_uid = Some("request-123".into()); + + assert_eq!( + event.base().metadata.correlation_uid.as_deref(), + Some("request-123") + ); + assert_eq!( + serde_json::to_value(&event).unwrap()["metadata"]["correlation_uid"], + "request-123" + ); + } + #[test] fn test_roundtrip_application_lifecycle() { let ctx = test_sandbox_context(); diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index 4b169e165c..35d949b160 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -74,5 +74,5 @@ pub use builders::{ pub use tracing_layers::{ OCSF_TARGET, OcsfJsonSettings, OcsfJsonlLayer, OcsfShorthandLayer, bind_thread_json_settings, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, - set_current_event, + register_event_enricher, set_current_event, }; diff --git a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs index ca5a5e9cdd..f5a8d41419 100644 --- a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs +++ b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs @@ -8,7 +8,11 @@ //! layers intercept this target, clone the event, and format it. //! After dispatch, `emit_ocsf_event` clears the thread-local. -use crate::events::OcsfEvent; +use std::sync::OnceLock; + +use crate::events::{BaseEventData, OcsfEvent}; + +static EVENT_ENRICHER: OnceLock = OnceLock::new(); std::thread_local! { // Thread-local storage for the current OCSF event being emitted. @@ -19,6 +23,27 @@ std::thread_local! { /// Target string used to identify OCSF tracing events. pub const OCSF_TARGET: &str = "ocsf"; +/// Register optional process-wide event enrichment at startup. +/// +/// The callback runs once per emission, before any output layer receives the +/// record, independently of trace sampling or export. It receives the common +/// base fields for every event class and should add only contextual fields, +/// preserving the record's content, classification, and timestamp. It must not +/// block, panic, or emit another OCSF event. +/// +/// Returns `false` if an enricher is already registered. Without an enricher, +/// emission does not add application-specific context. +pub fn register_event_enricher(enricher: fn(&mut BaseEventData)) -> bool { + EVENT_ENRICHER.set(enricher).is_ok() +} + +fn enrich(event: &mut OcsfEvent) { + crate::trace::enrich(event); + if let Some(enrich) = EVENT_ENRICHER.get() { + enrich(event.base_mut()); + } +} + /// Clone the current thread-local OCSF event, if any. /// /// Multiple layers can call this for the same event — each receives @@ -35,7 +60,7 @@ pub fn clone_current_event() -> Option { /// /// Both layers receive the event — `clone_current_event()` is non-consuming. pub fn emit_ocsf_event(mut event: OcsfEvent) { - crate::trace::enrich(&mut event); + enrich(&mut event); // Store the event in thread-local so layers can access it set_current_event(event); @@ -83,7 +108,7 @@ pub fn clear_current_event() { /// per-event `sandbox_id` field here instead. One tracing event feeds both the /// JSONL audit file and the routing bus. pub fn emit_ocsf_event_routed(sandbox_id: &str, mut event: OcsfEvent) { - crate::trace::enrich(&mut event); + enrich(&mut event); let message = event.format_shorthand(); set_current_event(event); tracing::info!(target: "ocsf", sandbox_id = %sandbox_id, message = %message); @@ -111,6 +136,22 @@ mod tests { use crate::events::{BaseEvent, OcsfEvent}; use crate::objects::{Metadata, Product}; + std::thread_local! { + // Registration is process-wide, but this test's callback must be inert + // for unrelated tests running in parallel or reusing this thread later. + static TEST_ENRICHER_SEQUENCE: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + fn test_enricher(event: &mut BaseEventData) { + TEST_ENRICHER_SEQUENCE.with(|sequence| { + if let Some(next) = sequence.get() { + sequence.set(Some(next + 1)); + event.metadata.correlation_uid = Some("emit-time-request".into()); + event.add_unmapped("request_sequence", next); + } + }); + } + fn test_event() -> OcsfEvent { OcsfEvent::Base(BaseEvent { base: BaseEventData::new( @@ -238,4 +279,78 @@ mod tests { "routed emit must match the bare path JSON" ); } + + #[test] + fn event_enrichment_runs_once_for_every_class_and_output_path() { + use crate::test_support::OcsfCapture; + use crate::tracing_layers::OcsfJsonlLayer; + use tracing_subscriber::prelude::*; + + assert!(register_event_enricher(test_enricher)); + assert!(!register_event_enricher(test_enricher)); + let capture = OcsfCapture::default(); + let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry() + .with(capture.clone()) + .with(OcsfJsonlLayer::new(SharedWriter(bytes.clone()))); + let context = crate::builders::test_sandbox_context(); + let mut expected = Vec::new(); + + tracing::subscriber::with_default(subscriber, || { + struct ResetEnricher; + impl Drop for ResetEnricher { + fn drop(&mut self) { + TEST_ENRICHER_SEQUENCE.with(|sequence| sequence.set(None)); + } + } + let plain = test_event(); + expected.push(plain.clone()); + emit_ocsf_event(plain); + + { + let _reset = ResetEnricher; + TEST_ENRICHER_SEQUENCE.with(|sequence| sequence.set(Some(1))); + let mut sequence = 1; + for routed in [false, true] { + for (_, mut event) in crate::builders::sample_events(&context) { + event.base_mut().add_unmapped("source_detail", "keep"); + event + .base_mut() + .add_unmapped("http_event_kind", "original_kind"); + let mut enriched = event.clone(); + enriched.base_mut().metadata.correlation_uid = + Some("emit-time-request".into()); + enriched + .base_mut() + .add_unmapped("request_sequence", sequence); + expected.push(enriched); + if routed { + emit_ocsf_event_routed("test-sandbox", event); + } else { + emit_ocsf_event(event); + } + assert!(clone_current_event().is_none()); + sequence += 1; + } + } + TEST_ENRICHER_SEQUENCE.with(|next| assert_eq!(next.get(), Some(sequence))); + } + + let plain = test_event(); + expected.push(plain.clone()); + emit_ocsf_event_routed("test-sandbox", plain); + }); + + assert_eq!(capture.events(), expected); + let expected_json: Vec<_> = expected + .iter() + .map(|event| event.to_json().unwrap()) + .collect(); + let output = String::from_utf8(bytes.lock().unwrap().clone()).unwrap(); + let json: Vec = output + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(json, expected_json); + } } diff --git a/crates/openshell-ocsf/src/tracing_layers/mod.rs b/crates/openshell-ocsf/src/tracing_layers/mod.rs index 808051b061..23e55ab305 100644 --- a/crates/openshell-ocsf/src/tracing_layers/mod.rs +++ b/crates/openshell-ocsf/src/tracing_layers/mod.rs @@ -13,7 +13,7 @@ mod shorthand_layer; pub use event_bridge::{ OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, - set_current_event, + register_event_enricher, set_current_event, }; pub use jsonl_layer::{OcsfJsonSettings, OcsfJsonlLayer, bind_thread_json_settings}; pub use shorthand_layer::OcsfShorthandLayer; diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 63489fde1d..cc4f6c44f7 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -34,6 +34,7 @@ const MAX_EXPORT_ATTEMPTS: usize = 4; const INITIAL_BACKOFF: Duration = Duration::from_millis(250); const MAX_BACKOFF: Duration = Duration::from_secs(2); const GAP_EVENT: &str = "openshell.ocsf.gap"; +const CORRELATION_UID_ATTRIBUTE: &str = "ocsf.metadata.correlation_uid"; /// Exports OCSF events to an OTLP collector as log records. pub struct OcsfOtlp { @@ -391,6 +392,9 @@ fn log_record( if let Some(uid) = &base.metadata.uid { record.add_attribute("ocsf.metadata.uid", uid.clone()); } + if let Some(uid) = &base.metadata.correlation_uid { + record.add_attribute(CORRELATION_UID_ATTRIBUTE, uid.clone()); + } if let Some(sandbox_id) = event.sandbox_id() { record.add_attribute("openshell.sandbox.id", sandbox_id.to_string()); } @@ -629,6 +633,19 @@ mod tests { assert_eq!(native(&event)["resources"][0]["uid"], "requested-sandbox"); } + #[test] + fn correlation_uid_is_a_stable_otlp_attribute() { + let mut event = event("sandbox-1", SeverityId::Informational); + let OcsfEvent::DeviceConfigStateChange(config) = &mut event else { + unreachable!("event fixture returns a configuration event"); + }; + config.base.metadata.correlation_uid = Some("request-123".into()); + assert_eq!( + attribute(&record(&event), CORRELATION_UID_ATTRIBUTE), + Some(AnyValue::String("request-123".into())) + ); + } + #[test] fn events_without_a_sandbox_carry_no_sandbox_attribute() { let record = record(&event("", SeverityId::Informational)); @@ -788,12 +805,27 @@ mod tests { .with(otlp.layer()); let span_context = tracing::subscriber::with_default(subscriber, || { let span = tracing::info_span!("decision"); - span.in_scope(|| ocsf_emit!(event("sandbox-1", SeverityId::Medium))); + let mut event = event("sandbox-1", SeverityId::Medium); + let OcsfEvent::DeviceConfigStateChange(config) = &mut event else { + unreachable!("event fixture returns a configuration event"); + }; + config.base.metadata.correlation_uid = + Some("request-with-or-without-sampling".into()); + span.in_scope(|| ocsf_emit!(event)); span.context().span().span_context().clone() }); otlp.shutdown().await; let records = exporter.exported(); + assert_eq!(records.len(), 1); + assert_eq!( + attribute(&records[0].record, CORRELATION_UID_ATTRIBUTE), + Some(AnyValue::String("request-with-or-without-sampling".into())) + ); + assert_eq!( + body(&records[0].record)["metadata"]["correlation_uid"], + "request-with-or-without-sampling" + ); let trace = records[0].record.trace_context(); if span_context.is_sampled() { let trace = trace.expect("sampled span context is attached"); diff --git a/crates/openshell-supervisor-middleware/src/headers.rs b/crates/openshell-supervisor-middleware/src/headers.rs index 883bc010c3..f926f16921 100644 --- a/crates/openshell-supervisor-middleware/src/headers.rs +++ b/crates/openshell-supervisor-middleware/src/headers.rs @@ -366,25 +366,31 @@ fn is_safe_value(value: &str) -> bool { } fn is_request_protected(name: &str) -> bool { - matches!( - name, - "authorization" - | "proxy-authorization" - | "proxy-authenticate" - | "cookie" - | "host" - | "content-length" - | "transfer-encoding" - | "connection" - | "proxy-connection" - | "keep-alive" - | "te" - | "trailer" - | "upgrade" - ) || name.starts_with("x-amz-") + is_platform_correlation_header(name) + || matches!( + name, + "authorization" + | "proxy-authorization" + | "proxy-authenticate" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-connection" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) + || name.starts_with("x-amz-") || name.starts_with("x-openshell-credential") } +fn is_platform_correlation_header(name: &str) -> bool { + name == "x-openshell-request-id" +} + /// Return whether a response header can carry authentication material and must /// never be exposed to middleware. #[must_use] @@ -402,7 +408,8 @@ pub fn is_response_credential_header(name: &str) -> bool { } fn is_response_protected(name: &str) -> bool { - is_response_credential_header(name) + is_platform_correlation_header(name) + || is_response_credential_header(name) || matches!( name, "connection" @@ -488,6 +495,42 @@ mod tests { ); } + #[test] + fn platform_request_id_is_immutable_for_all_http_phases() { + for authority in [ + HeaderAuthority::Request, + HeaderAuthority::HookRequest, + HeaderAuthority::RequestTrailers, + HeaderAuthority::Response, + HeaderAuthority::HookResponse, + HeaderAuthority::ResponseTrailers, + ] { + let name = "X-OpenShell-Request-ID"; + let write_error = apply( + authority, + &[], + &[], + &[write(name, "forged", ExistingHeaderAction::Overwrite)], + ) + .expect_err("platform request ID write"); + assert_eq!( + write_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + + let remove_error = apply(authority, &[], &[], &[remove(name)]) + .expect_err("platform request ID removal"); + assert_eq!( + remove_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + } + } + #[test] fn unsafe_header_value_is_rejected() { let error = apply( diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index e51f282cd5..8f000adb7c 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -38,6 +38,8 @@ hex = "0.4" hickory-proto = "0.26.1" ipnet = "2" miette = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } prost-types = { workspace = true } rcgen = { workspace = true } regorus = { version = "0.9", default-features = false, features = ["std", "arc", "glob"] } @@ -56,6 +58,7 @@ tokio-rustls = { workspace = true } tonic = { workspace = true } tower-mcp-types = { workspace = true } tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } uuid = { workspace = true } webpki-roots = { workspace = true, optional = true } diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 92257fb24a..dd8af76169 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -55,3 +55,42 @@ domain event and a detection finding without credential values or placeholders. Explicit body and WebSocket rewrite flags authorize those locations for ordinary static credentials. SigV4 signing retains the endpoint-scoped resolver's existing trusted access. + +## HTTP enforcement correlation + +Each inspected HTTP request has one UUID shared by middleware, OCSF +`metadata.correlation_uid`, and the protected `x-openshell-request-id` response +header. Application correlation must use another header. The receipt is only a +join key: it does not establish identity, authorization or complete delivery. +L4 and passthrough traffic do not provide this header guarantee. + +The emission hook adds request correlation to every OCSF class in the current +request scope, including findings. Policy, middleware, token-grant and finding +records are individual events with their own timestamps and classification. +`unmapped.http_event_kind` identifies decisions, findings, connection events, +middleware invocations and token grants. Middleware invocations use +`unmapped.middleware_phase` for request, response and WebSocket phases. +`unmapped.request_sequence` starts at 1 per request and describes local emission +order, not downstream delivery order. Count policy outcomes by selecting the +actual policy/firewall control record within a receipt group, not every +`decision` record. + +Request IDs and audit outcomes do not depend on OpenTelemetry, sampling or +collector availability. Inspected requests have a supervisor-owned request span +and children around stages that actually run, with valid workload W3C context +only as an untrusted link. Their records never create an extra denial-only span, +even if the request span is disabled or unsampled. Uninspected HTTP denials use +bounded `supervisor.egress.deny` spans. The ambient HTTP audit scope ends before +protocol-start validation; one-shot attribution joins the existing upgrade +handoff record to the handshake and releases the request span before frames. +Later WebSocket frames do not reuse the handshake receipt. Terminal errors carry their request context explicitly, +and policy-reload cancellation records the active request before dropping it. +Idle connection closures have no HTTP receipt. +The request span ends after its owned stages, not when response headers are logged. +The `openshell.upstream.send` stage measures request writes and flush; response +handling is measured separately. Handled middleware failures remain stage errors +even when the local rejection response is delivered successfully. Receipt +filtering preserves raw HTTP bytes +and borrows unchanged header blocks. It removes only protected receipt fields +and their `Connection` or `Trailer` nominations, including folded continuations, +without adding general response-header validation. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs new file mode 100644 index 0000000000..1d4cfe36fa --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -0,0 +1,486 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Request receipts and emission-time correlation without changing audit records. + +use std::fmt; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use openshell_ocsf::OcsfEvent; +use tracing::Instrument as _; + +use crate::l7::provider::L7Request; +use crate::l7::relay::L7EvalContext; +pub use crate::l7::request_trace::RequestStage; +use crate::l7::request_trace::RequestTrace; + +tokio::task_local! { + static CURRENT_REQUEST: Arc>>>; +} + +/// Give each connection its own request slot, including across task migration. +pub fn scope_connection(future: F) -> impl Future { + #[cfg(test)] + init_request_correlation().expect("request correlation enricher registration"); + let slot = CURRENT_REQUEST + .try_with(Arc::clone) + .unwrap_or_else(|_| Arc::new(Mutex::new(None))); + CURRENT_REQUEST.scope(slot, future) +} + +/// Install the network supervisor's process-wide audit enrichment at startup. +/// +/// A different enricher is a configuration conflict, not a reason to silently +/// emit inspected records without their request receipt. +pub fn init_request_correlation() -> miette::Result<()> { + static REGISTRATION: OnceLock> = OnceLock::new(); + match REGISTRATION.get_or_init(|| { + openshell_ocsf::register_event_enricher(enrich_request) + .then_some(()) + .ok_or("cannot initialize HTTP request correlation: an OCSF event enricher is already registered") + }) { + Ok(()) => Ok(()), + Err(message) => Err(miette::miette!("{message}")), + } +} + +pub fn is_inspected_request() -> bool { + current_request().is_some_and(|scope| scope.inspected) +} + +pub fn current_request_span() -> Option { + current_request() + .filter(|scope| scope.inspected) + .map(|scope| scope.span()) +} + +fn current_request() -> Option> { + CURRENT_REQUEST + .try_with(|current| { + current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + }) + .ok() + .flatten() +} + +fn in_request_scope(scope: &Arc, operation: impl FnOnce() -> T) -> T { + if current_request().is_some_and(|current| Arc::ptr_eq(¤t, scope)) { + operation() + } else { + let slot = Arc::new(Mutex::new(Some(Arc::clone(scope)))); + CURRENT_REQUEST.sync_scope(slot, operation) + } +} + +pub fn record_current_failure(outcome: HttpOutcome, reason: &'static str) { + if let Some(scope) = current_request().filter(|scope| scope.inspected) { + scope.record_failure(outcome, reason); + } +} + +fn enrich_request(event: &mut openshell_ocsf::BaseEventData) { + let Some(scope) = current_request().filter(|scope| scope.inspected) else { + return; + }; + event.metadata.correlation_uid = Some(scope.request_id.clone()); + let sequence = scope.sequence.fetch_add(1, Ordering::Relaxed); + let event_kind = default_event_kind(event.class_uid); + let fields = event.unmapped.get_or_insert_with(|| serde_json::json!({})); + if let Some(fields) = fields.as_object_mut() { + fields + .entry("http_event_kind") + .or_insert_with(|| event_kind.into()); + fields.insert("request_sequence".into(), sequence.into()); + } +} + +fn default_event_kind(class_uid: u32) -> &'static str { + match class_uid { + 4002 => "decision", + 2004 => "finding", + 4001 | 4007 => "connection", + 1007 | 6002 => "lifecycle", + 5019 => "configuration", + 3002 => "authentication", + _ => "event", + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpOutcome { + Allowed, + Denied, + PlatformError, + UpstreamError, + Upgraded, +} + +#[derive(Debug)] +struct RequestScope { + request_id: String, + inspected: bool, + sequence: AtomicU64, + trace: Mutex>, +} + +impl RequestScope { + fn span(&self) -> tracing::Span { + self.trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map_or_else(tracing::Span::none, RequestTrace::span) + } + + fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { + if let Some(trace) = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_outcome(outcome); + trace.record_failure_reason(reason); + } + } +} + +/// Carry request attribution only with an actual terminal error, never as a +/// connection-wide "last request" that could misattribute unrelated events. +#[derive(Debug)] +struct CapturedRequestError { + scope: Arc, + span: tracing::Span, + detail: String, +} + +impl fmt::Display for CapturedRequestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.detail) + } +} + +/// Emit the existing terminal record in the request that returned the error, +/// even after its request guard has restored the connection's ambient scope. +pub fn emit_error_event(event: OcsfEvent, error: &miette::Report) { + if let Some(captured) = error.downcast_ref::() { + in_request_scope(&captured.scope, || { + crate::telemetry::emit_inspected_decision([event], Some(&captured.span)); + }); + } else { + crate::telemetry::emit_decision([event], None); + } +} + +/// Retain attribution for one explicit handoff record, not ambient protocol +/// traffic. The request span stays alive only until this value emits or drops. +#[derive(Debug)] +pub struct DetachedRequestEvent { + scope: Arc, + span: tracing::Span, +} + +impl DetachedRequestEvent { + pub(crate) fn emit(self, event: OcsfEvent) { + in_request_scope(&self.scope, || { + crate::telemetry::emit_inspected_decision([event], Some(&self.span)); + }); + } +} + +#[derive(Debug)] +pub struct RequestCorrelation { + scope: Arc, + // Request loops are sequential; nested scopes restore in LIFO order. + previous: Option>, +} + +/// An interrupted stage is an error even if its future never returns a result. +struct StageResult { + span: tracing::Span, + completed: bool, +} + +impl StageResult { + fn record(&mut self, failure_reason: Option<&'static str>) { + self.completed = true; + if let Some(reason) = failure_reason { + self.span.record("otel.status_code", "ERROR"); + self.span.record("openshell.request.failure_reason", reason); + } else { + self.span.record("otel.status_code", "OK"); + } + } +} + +impl Drop for StageResult { + fn drop(&mut self) { + if !self.completed { + self.span.record("otel.status_code", "ERROR"); + self.span + .record("openshell.request.failure_reason", "cancelled"); + } + } +} + +impl RequestCorrelation { + pub(crate) fn new(request: &L7Request, ctx: &L7EvalContext) -> Self { + Self::new_http(&request.action, &request.raw_header, ctx) + } + + pub(crate) fn new_http(method: &str, raw_header: &[u8], ctx: &L7EvalContext) -> Self { + Self::build(method, raw_header, ctx, uuid::Uuid::new_v4().to_string()) + } + + pub(crate) fn uninspected() -> Self { + Self::install(RequestScope { + request_id: uuid::Uuid::new_v4().to_string(), + inspected: false, + sequence: AtomicU64::new(1), + trace: Mutex::new(None), + }) + } + + #[cfg(test)] + pub(crate) fn with_request_id( + request: &L7Request, + ctx: &L7EvalContext, + request_id: String, + ) -> Self { + init_request_correlation().expect("request correlation enricher registration"); + Self::build(&request.action, &request.raw_header, ctx, request_id) + } + + fn build(method: &str, raw_header: &[u8], ctx: &L7EvalContext, request_id: String) -> Self { + let trace = RequestTrace::new(&request_id, method, &ctx.host, ctx.port, raw_header); + Self::install(RequestScope { + request_id, + inspected: true, + sequence: AtomicU64::new(1), + trace: Mutex::new(Some(trace)), + }) + } + + fn install(scope: RequestScope) -> Self { + let scope = Arc::new(scope); + let previous = CURRENT_REQUEST + .try_with(|current| { + current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .replace(Arc::clone(&scope)) + }) + .ok() + .flatten(); + Self { scope, previous } + } + + pub(crate) fn request_id(&self) -> &str { + &self.scope.request_id + } + + pub(crate) fn capture_error(&self, error: miette::Report) -> miette::Report { + if !self.scope.inspected || error.downcast_ref::().is_some() { + return error; + } + if let Some(trace) = self + .scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_unclassified_failure(); + } + let captured = CapturedRequestError { + scope: Arc::clone(&self.scope), + span: self.scope.span(), + detail: error.to_string(), + }; + error.wrap_err(captured) + } + + pub(crate) fn stage(&self, stage: RequestStage) -> tracing::Span { + self.scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map_or_else(tracing::Span::none, |trace| trace.stage(stage)) + } + + /// Record a stage's actual result without exporting its potentially + /// sensitive error value or changing cancellation behavior. + pub(crate) fn run_stage( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + future: impl Future>, + ) -> impl Future> { + self.run_stage_classified(stage, failure_outcome, reason, future, |_| None) + } + + /// Include handled failures returned as successful transport results. + pub(crate) fn run_stage_classified( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + future: impl Future>, + classify_failure: impl FnOnce(&T) -> Option<&'static str>, + ) -> impl Future> { + let mut stage = StageResult { + span: self.stage(stage), + completed: false, + }; + let future = future.instrument(stage.span.clone()); + async move { + let result = future.await; + let failure_reason = result.as_ref().map_or(Some(reason), classify_failure); + stage.record(failure_reason); + if let Some(reason) = failure_reason { + self.record_failure(failure_outcome, reason); + } + result + } + } + + pub(crate) fn run_stage_sync( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + operation: impl FnOnce() -> Result, + ) -> Result { + let mut stage = StageResult { + span: self.stage(stage), + completed: false, + }; + let result = stage.span.in_scope(operation); + stage.record(result.is_err().then_some(reason)); + if result.is_err() { + self.record_failure(failure_outcome, reason); + } + result + } + + /// End HTTP tracing and restore the previous ambient scope before handing + /// an upgraded connection to its longer-lived protocol relay. + pub(crate) fn finish_http_request(self) { + drop(self); + } + + /// Restore the ambient scope now, retaining the request span only for the + /// existing upgrade handoff record after protocol-start validation. + pub(crate) fn finish_http_request_for_event(self) -> Option { + let event = self.scope.inspected.then(|| DetachedRequestEvent { + scope: Arc::clone(&self.scope), + span: self.scope.span(), + }); + self.finish_http_request(); + event + } + + fn finish_http_trace(&self) { + let trace = self + .scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(trace); + } + + pub(crate) fn record_outcome(&self, outcome: HttpOutcome) { + if let Some(trace) = self + .scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_outcome(outcome); + } + } + + pub(crate) fn record_response_status(&self, status: u16) { + if let Some(trace) = self + .scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_status(status); + if status == 101 { + trace.record_outcome(HttpOutcome::Upgraded); + } + } + } + + pub(crate) fn record_rejection( + &self, + status: u16, + outcome: HttpOutcome, + reason: Option<&'static str>, + ) { + self.record_outcome(outcome); + self.record_response_status(status); + if let Some(reason) = reason { + self.record_failure(outcome, reason); + } + } + + pub(crate) fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { + self.scope.record_failure(outcome, reason); + } + + pub(crate) fn emit(&self, event: OcsfEvent) { + if !self.scope.inspected { + crate::telemetry::emit_decision([event], None); + return; + } + in_request_scope(&self.scope, || { + crate::telemetry::emit_inspected_decision([event], Some(&self.scope.span())); + }); + } +} + +pub fn emit_request_events( + correlation: Option<&RequestCorrelation>, + events: impl IntoIterator, +) { + if let Some(correlation) = correlation { + for event in events { + correlation.emit(event); + } + } else { + crate::telemetry::emit_decision(events, None); + } +} + +impl Drop for RequestCorrelation { + fn drop(&mut self) { + self.finish_http_trace(); + let _ = CURRENT_REQUEST.try_with(|current| { + let mut current = current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current + .as_ref() + .is_some_and(|scope| Arc::ptr_eq(scope, &self.scope)) + { + *current = self.previous.take(); + } + }); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs new file mode 100644 index 0000000000..b7c7dca6cb --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -0,0 +1,722 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use openshell_ocsf::test_support::OcsfCapture; +use openshell_ocsf::{ + ActionId, ActivityId, DetectionFindingBuilder, DispositionId, HttpActivityBuilder, HttpRequest, + HttpResponse, NetworkActivityBuilder, SeverityId, StatusId, +}; +use tracing::instrument::WithSubscriber as _; + +fn request() -> L7Request { + init_request_correlation().unwrap(); + L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: std::collections::HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + } +} + +fn context() -> L7EvalContext { + L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + } +} + +fn allowed() -> OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) + .firewall_rule("protected-api", "l7") + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .status(StatusId::Success) + .message("HTTP request admitted by policy") + .build() +} + +fn deny() -> OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) + .http_response(HttpResponse { code: 403 }) + .firewall_rule("content-guard", "supervisor-middleware") + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .status(StatusId::Failure) + .severity(SeverityId::Medium) + .message("middleware blocked request") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "response") + .unmapped("transformed", true) + .build() +} + +#[test] +fn records_emit_immediately_and_keep_their_original_meaning() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + for (index, source) in [allowed(), deny(), deny()].into_iter().enumerate() { + let mut expected = source.to_json().unwrap(); + expected["metadata"]["correlation_uid"] = correlation.request_id().into(); + expected["unmapped"]["request_sequence"] = (index + 1).into(); + if index == 0 { + expected["unmapped"]["http_event_kind"] = "decision".into(); + } + correlation.emit(source); + assert_eq!(capture.json_events()[index], expected); + correlation.record_response_status(502); + } + drop(correlation); + // No replacement, fallback, or late-failure suppression. + assert_eq!(capture.json_events().len(), 3); +} + +#[test] +fn unused_request_scope_does_not_create_an_audit_record() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + drop(RequestCorrelation::new(&request(), &context())); + assert!(capture.json_events().is_empty()); +} + +#[test] +fn request_join_does_not_depend_on_export_and_never_creates_denial_spans() { + use opentelemetry_sdk::trace::Sampler; + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.emit(allowed()); + correlation.emit(deny()); + }); + let records = capture.events.json_events(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["action"], "Allowed"); + assert_eq!(records[1]["action"], "Denied"); + assert_eq!( + records[0]["metadata"]["correlation_uid"], + records[1]["metadata"]["correlation_uid"] + ); + assert!(records[0]["metadata"]["correlation_uid"].is_string()); + assert_eq!(records[0]["trace"]["uid"].is_string(), sampled); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + capture.shutdown(); + } +} + +#[test] +fn audit_events_do_not_override_the_actual_response_outcome() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_response_status(200); + correlation.record_outcome(HttpOutcome::Allowed); + correlation.emit(deny()); + }); + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "allowed" + ); + assert_eq!( + span_attribute(root, "http.response.status_code") + .unwrap() + .as_str(), + "200" + ); + assert_eq!(capture.events.json_events()[0]["action"], "Denied"); + capture.shutdown(); +} + +#[tokio::test] +async fn ambient_scope_stamps_companions_without_call_site_wiring() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + crate::telemetry::emit_decision( + [ + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .message("credential refused") + .dst_endpoint(openshell_ocsf::Endpoint::from_domain("example.test", 443)) + .build(), + DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .finding_info(openshell_ocsf::FindingInfo::new( + "credential", + "Credential refused", + )) + .build(), + ], + None, + ); + let records = capture.json_events(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["unmapped"]["http_event_kind"], "connection"); + assert_eq!(records[1]["unmapped"]["http_event_kind"], "finding"); + for (index, record) in records.iter().enumerate() { + assert_eq!( + record["metadata"]["correlation_uid"], + correlation.request_id() + ); + assert_eq!(record["unmapped"]["request_sequence"], index + 1); + } + drop(correlation); + crate::telemetry::emit_decision([allowed()], None); + assert!(capture.json_events()[2]["metadata"]["correlation_uid"].is_null()); + }) + .await; +} + +#[tokio::test] +async fn concurrent_connections_and_nested_requests_restore_their_own_scope() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let run = |name: &'static str| { + scope_connection(async move { + let outer = RequestCorrelation::with_request_id(&request(), &context(), name.into()); + crate::telemetry::emit_decision([allowed()], None); + tokio::task::yield_now().await; + { + let _nested = RequestCorrelation::with_request_id( + &request(), + &context(), + format!("{name}-nested"), + ); + crate::telemetry::emit_decision([allowed()], None); + } + crate::telemetry::emit_decision([allowed()], None); + drop(outer); + let _uninspected = RequestCorrelation::uninspected(); + crate::telemetry::emit_decision([allowed()], None); + }) + }; + tokio::join!(run("first"), run("second")); + let records = capture.json_events(); + assert_eq!(records.len(), 8); + for name in ["first", "second"] { + let outer: Vec<_> = records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == name) + .collect(); + assert_eq!(outer.len(), 2); + assert_eq!(outer[0]["unmapped"]["request_sequence"], 1); + assert_eq!(outer[1]["unmapped"]["request_sequence"], 2); + assert_eq!( + records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == format!("{name}-nested")) + .count(), + 1 + ); + } + assert_eq!( + records + .iter() + .filter(|record| record["metadata"]["correlation_uid"].is_null()) + .count(), + 2 + ); + assert!(!is_inspected_request()); +} + +#[tokio::test] +async fn cancelling_a_request_does_not_leak_its_context() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let (ready, started) = tokio::sync::oneshot::channel(); + let mut pending = Box::pin(scope_connection(async { + let _correlation = RequestCorrelation::new(&request(), &context()); + ready.send(()).unwrap(); + std::future::pending::<()>().await; + })); + tokio::select! { + () = &mut pending => unreachable!(), + _ = started => {}, + } + assert!(!is_inspected_request()); + // Drop while pending, then start a new connection on the same task. + drop(pending); + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + crate::telemetry::emit_decision([allowed()], None); + assert_eq!( + capture.json_events()[0]["metadata"]["correlation_uid"], + correlation.request_id() + ); + assert_eq!(capture.json_events()[0]["unmapped"]["request_sequence"], 1); + }) + .await; +} + +#[test] +fn event_kinds_distinguish_decisions_from_companions() { + for (class_uid, expected) in [ + (4002, "decision"), + (2004, "finding"), + (4001, "connection"), + (4007, "connection"), + (1007, "lifecycle"), + (6002, "lifecycle"), + (5019, "configuration"), + (3002, "authentication"), + (6003, "event"), + (0, "event"), + ] { + assert_eq!(default_event_kind(class_uid), expected); + } +} + +#[tokio::test] +async fn finishing_http_restores_the_previous_scope_before_protocol_relay() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let outer = RequestCorrelation::with_request_id(&request(), &context(), "outer".into()); + let nested = RequestCorrelation::with_request_id(&request(), &context(), "nested".into()); + crate::telemetry::emit_decision([allowed()], None); + + nested.finish_http_request(); + crate::telemetry::emit_decision([allowed()], None); + assert!(is_inspected_request()); + + outer.finish_http_request(); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + tokio::task::yield_now().await; + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let records = capture.json_events(); + assert_eq!(records[0]["metadata"]["correlation_uid"], "nested"); + assert_eq!(records[1]["metadata"]["correlation_uid"], "outer"); + assert_eq!(records[1]["unmapped"]["request_sequence"], 1); + assert!(records[2]["metadata"]["correlation_uid"].is_null()); + assert!(records[2]["unmapped"]["request_sequence"].is_null()); +} + +#[tokio::test] +async fn detached_handoff_preserves_only_its_explicit_record_and_ends_before_frames() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + scope_connection(async { + let correlation = + RequestCorrelation::with_request_id(&request(), &context(), "handshake".into()); + correlation.emit(allowed()); + correlation.record_response_status(101); + let handoff = correlation.finish_http_request_for_event().unwrap(); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + + // Protocol-start work must not inherit the detached HTTP context. + crate::telemetry::emit_decision([allowed()], None); + tokio::task::yield_now().await; + handoff.emit(allowed()); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + + // The retained root is exported before any longer-lived frame relay. + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "upgraded" + ); + crate::telemetry::emit_decision([allowed()], None); + }) + .with_subscriber(capture.subscriber()) + .await; + let events = capture.events.json_events(); + assert_eq!(events.len(), 4); + assert_eq!(events[0]["metadata"]["correlation_uid"], "handshake"); + assert_eq!(events[0]["unmapped"]["request_sequence"], 1); + assert_eq!(events[2]["metadata"]["correlation_uid"], "handshake"); + assert_eq!(events[2]["unmapped"]["request_sequence"], 2); + assert!(events[0]["trace"]["uid"].is_string()); + assert_eq!(events[0]["trace"], events[2]["trace"]); + for index in [1, 3] { + assert!(events[index]["metadata"]["correlation_uid"].is_null()); + assert!(events[index]["unmapped"]["request_sequence"].is_null()); + assert!(events[index]["unmapped"]["http_event_kind"].is_null()); + assert!(events[index]["trace"].is_null()); + } + assert_eq!(capture.created_span_names(), ["openshell.http.request"]); + capture.shutdown(); +} + +#[tokio::test] +async fn cancelling_a_detached_handoff_ends_the_root_without_emitting_or_leaking() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + let mut pending = Box::pin( + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_response_status(101); + let handoff = correlation.finish_http_request_for_event().unwrap(); + assert!(!is_inspected_request()); + std::future::pending::<()>().await; + handoff.emit(allowed()); + }) + .with_subscriber(capture.subscriber()), + ); + assert!(futures::poll!(&mut pending).is_pending()); + assert!(!is_inspected_request()); + drop(pending); + assert_eq!(capture.spans().len(), 1); + assert!(capture.events.json_events().is_empty()); + tracing::subscriber::with_default(capture.subscriber(), || { + crate::telemetry::emit_decision([allowed()], None); + }); + let events = capture.events.json_events(); + assert_eq!(events.len(), 1); + assert!(events[0]["metadata"]["correlation_uid"].is_null()); + assert!(events[0]["unmapped"]["request_sequence"].is_null()); + assert!(events[0]["trace"].is_null()); + capture.shutdown(); +} + +#[tokio::test] +async fn an_uninspected_handoff_has_no_detached_attribution() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let correlation = RequestCorrelation::uninspected(); + assert!(correlation.finish_http_request_for_event().is_none()); + assert!(!is_inspected_request()); + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let events = capture.json_events(); + assert_eq!(events.len(), 1); + assert!(events[0]["metadata"]["correlation_uid"].is_null()); + assert!(events[0]["unmapped"]["request_sequence"].is_null()); + assert!(events[0]["unmapped"]["http_event_kind"].is_null()); +} + +#[tokio::test] +async fn stage_results_record_bounded_errors_and_preserve_a_failed_root() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + scope_connection(async { + let correlation = RequestCorrelation::new_http( + "POST", + b"POST / HTTP/1.1\r\nX-Opaque: \xff\r\n\r\n", + &context(), + ); + let success = correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || Ok::<_, &str>(42), + ); + assert_eq!(success, Ok(42)); + let failure = correlation + .run_stage( + RequestStage::UpstreamDial, + HttpOutcome::UpstreamError, + "upstream_connect_failed", + async { Err::<(), _>("DO-NOT-RECORD-ERROR") }, + ) + .await; + assert_eq!(failure, Err("DO-NOT-RECORD-ERROR")); + correlation.record_failure(HttpOutcome::PlatformError, "request_failed"); + correlation.record_outcome(HttpOutcome::Allowed); + }) + .with_subscriber(capture.subscriber()) + .await; + let spans = capture.spans(); + let policy = spans + .iter() + .find(|span| span.name == "openshell.policy.evaluate") + .unwrap(); + assert_eq!(policy.status, Status::Ok); + let dial = spans + .iter() + .find(|span| span.name == "openshell.upstream.connect") + .unwrap(); + assert!(matches!(dial.status, Status::Error { .. })); + assert_eq!( + span_attribute(dial, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert!(matches!(root.status, Status::Error { .. })); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + assert_eq!( + span_attribute(root, "http.request.method") + .unwrap() + .as_str(), + "POST" + ); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD-ERROR")); + capture.shutdown(); +} + +#[tokio::test] +async fn a_cancelled_stage_drops_its_future_and_closes_with_a_bounded_reason() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + use std::sync::atomic::AtomicBool; + + struct Dropped(Arc); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::Relaxed); + } + } + + let dropped = Arc::new(AtomicBool::new(false)); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + let mut pending = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + async { + let _drop = Dropped(Arc::clone(&dropped)); + std::future::pending::>().await + }, + )); + assert!(futures::poll!(&mut pending).is_pending()); + drop(pending); + assert!(dropped.load(Ordering::Relaxed)); + }) + .with_subscriber(capture.subscriber()) + .await; + assert!(!is_inspected_request()); + let spans = capture.spans(); + let response = spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert!(matches!(response.status, Status::Error { .. })); + assert_eq!( + span_attribute(response, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "cancelled" + ); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert!(matches!(root.status, Status::Error { .. })); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "incomplete" + ); + capture.shutdown(); +} + +#[tokio::test] +async fn captured_terminal_errors_keep_request_attribution_after_the_guard_drops() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + scope_connection(async { + let correlation = RequestCorrelation::with_request_id( + &request(), + &context(), + "failed-request".into(), + ); + correlation.emit(allowed()); + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); + let original = miette::miette!("original transfer error").wrap_err("transfer failed"); + let original_chain: Vec<_> = original.chain().map(ToString::to_string).collect(); + let captured = correlation.capture_error(original); + assert_eq!(captured.to_string(), "transfer failed"); + assert_eq!( + captured + .chain() + .skip(1) + .map(ToString::to_string) + .collect::>(), + original_chain + ); + let captured = correlation.capture_error(captured); + assert_eq!(captured.chain().count(), original_chain.len() + 1); + drop(correlation); + assert!(!is_inspected_request()); + + let next = + RequestCorrelation::with_request_id(&request(), &context(), "next-request".into()); + emit_error_event(deny(), &captured); + emit_error_event( + DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .finding_info(openshell_ocsf::FindingInfo::new( + "transfer-failed", + "Transfer failed", + )) + .build(), + &captured, + ); + crate::telemetry::emit_decision([allowed()], None); + drop(next); + }) + .with_subscriber(capture.subscriber()) + .await; + let events = capture.events.json_events(); + assert_eq!(events.len(), 4); + for (index, event) in events[..3].iter().enumerate() { + assert_eq!(event["metadata"]["correlation_uid"], "failed-request"); + assert_eq!(event["unmapped"]["request_sequence"], index + 1); + } + assert_eq!(events[1]["trace"]["uid"].is_string(), sampled); + assert_eq!(events[2]["unmapped"]["http_event_kind"], "finding"); + assert_eq!(events[3]["metadata"]["correlation_uid"], "next-request"); + assert_eq!(events[3]["unmapped"]["request_sequence"], 1); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + if sampled { + let spans = capture.spans(); + let failed = spans + .iter() + .find(|span| { + span_attribute(span, "openshell.request.id") + .is_some_and(|id| id.as_str() == "failed-request") + }) + .unwrap(); + assert!(matches!(failed.status, Status::Error { .. })); + assert_eq!( + span_attribute(failed, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_send_failed" + ); + } + capture.shutdown(); + } +} + +#[test] +fn failed_denial_delivery_marks_the_root_failed_without_replacing_the_reason() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_failure(HttpOutcome::Denied, "request_middleware_denied"); + let error = correlation.capture_error(miette::miette!("response write failed")); + drop(correlation); + drop(error); + }); + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "request_middleware_denied" + ); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + capture.shutdown(); +} + +#[test] +fn an_unscoped_terminal_error_does_not_borrow_a_finished_request() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let error = correlation.capture_error(miette::miette!("request error")); + drop(correlation); + emit_error_event(allowed(), &error); + emit_error_event(allowed(), &miette::miette!("unrelated connection error")); + let records = capture.json_events(); + assert!(records[0]["metadata"]["correlation_uid"].is_string()); + assert!(records[1]["metadata"]["correlation_uid"].is_null()); + assert!(records[1]["unmapped"]["request_sequence"].is_null()); +} + +#[tokio::test] +async fn nested_connection_scopes_share_the_active_request_with_a_sibling_watcher() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let (ready, started) = tokio::sync::oneshot::channel(); + let relay = scope_connection(async { + let _request = RequestCorrelation::with_request_id( + &request(), + &context(), + "active-request".into(), + ); + ready.send(()).unwrap(); + std::future::pending::<()>().await; + }); + let watcher = async { + started.await.unwrap(); + assert!(is_inspected_request()); + record_current_failure(HttpOutcome::Denied, "policy_reloaded"); + crate::telemetry::emit_decision([deny()], None); + }; + tokio::select! { + () = relay => unreachable!(), + () = watcher => {}, + } + assert!(!is_inspected_request()); + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let records = capture.json_events(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["metadata"]["correlation_uid"], "active-request"); + assert_eq!(records[0]["unmapped"]["request_sequence"], 1); + assert!(records[1]["metadata"]["correlation_uid"].is_null()); +} diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index e688711ad1..52c6dbe967 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -24,6 +24,9 @@ use std::sync::{Arc, OnceLock}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::sync::mpsc; +use tracing::Instrument as _; + +use super::correlation::{RequestStage, emit_request_events}; /// How long the sandbox may stop sending a request body while a DUPLEX /// request middleware session is held. DUPLEX has no total deadline, so this @@ -210,6 +213,7 @@ impl HttpMiddlewareExchange { } } + #[cfg(test)] pub async fn apply_request( &self, request: crate::l7::provider::L7Request, @@ -221,18 +225,36 @@ impl HttpMiddlewareExchange { where C: AsyncRead + AsyncWrite + Unpin + Send, { - Box::pin(self.apply_request_with_delivery( + self.apply_request_correlated(request, client, ctx, scheme, transformed_body_policy, None) + .await + } + + pub(crate) async fn apply_request_correlated( + &self, + request: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + correlation: Option<&super::correlation::RequestCorrelation>, + ) -> Result + where + C: AsyncRead + AsyncWrite + Unpin + Send, + { + Box::pin(self.apply_request_with_delivery_correlated( request, client, ctx, scheme, transformed_body_policy, RequestBodyDelivery::Incremental, + correlation, )) .await } - pub async fn apply_request_with_delivery( + #[allow(clippy::too_many_arguments)] + pub(crate) async fn apply_request_with_delivery_correlated( &self, request: crate::l7::provider::L7Request, client: &mut C, @@ -240,12 +262,13 @@ impl HttpMiddlewareExchange { scheme: &str, transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin + Send, { let result = Box::pin( - apply_middleware_chain_for_scheme_with_request_id_and_delivery( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( request, client, ctx, @@ -256,6 +279,7 @@ impl HttpMiddlewareExchange { transformed_body_policy, &self.request_id, delivery, + correlation, ), ) .await?; @@ -393,11 +417,13 @@ pub fn emit_websocket_preflight_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, ) { - emit_websocket_invocations(ctx, &outcome.invocations); - emit_websocket_coverage(ctx, &outcome.coverage); - for event in websocket_preflight_finding_events(outcome) { - egress_emit!(event); - } + emit_websocket_invocations( + ctx, + &outcome.invocations, + WebSocketMiddlewarePhase::Preflight, + ); + crate::telemetry::emit_decision(websocket_coverage_events(ctx, &outcome.coverage), None); + crate::telemetry::emit_decision(websocket_preflight_finding_events(outcome), None); if outcome.saturated { emit_websocket_saturation(ctx); } @@ -469,14 +495,18 @@ pub(super) fn emit_websocket_session_start_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketSessionStartOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + emit_websocket_invocations( + ctx, + &outcome.invocations, + WebSocketMiddlewarePhase::SessionStart, + ); } pub(super) fn emit_websocket_message_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketMessageOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + emit_websocket_invocations(ctx, &outcome.invocations, WebSocketMiddlewarePhase::Message); if outcome.saturated { emit_websocket_saturation(ctx); } @@ -529,9 +559,27 @@ pub(super) fn middleware_finding_events( .collect() } +#[derive(Clone, Copy)] +enum WebSocketMiddlewarePhase { + Preflight, + SessionStart, + Message, +} + +impl WebSocketMiddlewarePhase { + const fn as_str(self) -> &'static str { + match self { + Self::Preflight => "websocket_preflight", + Self::SessionStart => "websocket_session_start", + Self::Message => "websocket_message", + } + } +} + fn emit_websocket_invocations( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], + phase: WebSocketMiddlewarePhase, ) { for invocation in invocations { use openshell_supervisor_middleware::WebSocketInvocationOutcome as Outcome; @@ -595,6 +643,8 @@ fn emit_websocket_invocations( .status(status) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", phase.as_str()) .message(format!( "WEBSOCKET_MIDDLEWARE {outcome_name} config={} implementation={} sequence={sequence} input_bytes={} replacement_bytes={replacement_size} transformed={} reason_code={reason_code}", invocation.config_name, @@ -708,10 +758,6 @@ fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf:: .build() } -fn emit_middleware_admission_exhausted(ctx: &L7EvalContext) { - egress_emit!(middleware_admission_exhausted_event(ctx)); -} - /// Largest body-buffering limit across the entries that actually resolved to a /// registered binding. Buffering for the most capable stage lets every stage /// that can handle the body run; stages whose own limit is smaller are failed @@ -731,6 +777,7 @@ pub(super) fn middleware_chain_body_limit( .max() } +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub async fn apply_middleware_chain_with_request_id( req: crate::l7::provider::L7Request, @@ -756,6 +803,7 @@ pub async fn apply_middleware_chain_with_request_id, request_id: &str, +) -> Result { + apply_middleware_chain_for_scheme_with_request_id_correlated( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_with_request_id_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + correlation: Option<&super::correlation::RequestCorrelation>, +) -> Result { + apply_middleware_chain_for_scheme_with_request_id_correlated( + req, + client, + ctx, + ctx.request_scheme(), + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + correlation, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_for_scheme_with_request_id_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { Box::pin( - apply_middleware_chain_for_scheme_with_request_id_and_delivery( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( req, client, ctx, @@ -813,14 +921,46 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< transformed_body_policy, request_id, RequestBodyDelivery::Incremental, + correlation, + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_with_request_id_and_delivery_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, +) -> Result { + Box::pin( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( + req, + client, + ctx, + ctx.request_scheme(), + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + correlation, ), ) .await } -/// Describe the request chain and run it on the engine of its HTTP hook version. -/// A chain that mixes entries of both HTTP hook versions fails -/// closed. +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< C: AsyncRead + AsyncWrite + Unpin + Send, @@ -835,62 +975,132 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, request_id: &str, delivery: RequestBodyDelivery, +) -> Result { + Box::pin( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + None, + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); } let chain = runner.describe_chain(&chain).await?; - match openshell_supervisor_middleware::chain_http_hook_version(&chain) { - ChainHttpHookVersion::Legacy => { - Box::pin(apply_protocol1_middleware_chain( - req, - client, - ctx, - scheme, - chain, - runner, - generation_guard, - transformed_body_policy, - request_id, - )) - .await - } - ChainHttpHookVersion::Mixed => { - emit_mixed_protocol_denial(ctx, &req, &chain); - Ok(MiddlewareApplyResult::Denied { - denial: None, - failure: Some(MiddlewareFailureDetail { - config_name: None, - code: HttpFailureCode::HookVersionsMixed, - }), - }) + let has_stages = !chain.is_empty(); + let apply = async { + let span = if has_stages && correlation.is_some() { + tracing::Span::current() + } else { + tracing::Span::none() + }; + match openshell_supervisor_middleware::chain_http_hook_version(&chain) { + ChainHttpHookVersion::Legacy => { + Box::pin(apply_protocol1_middleware_chain( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + )) + .await + } + ChainHttpHookVersion::Mixed => { + emit_mixed_protocol_denial(ctx, &req, &chain); + Ok(MiddlewareApplyResult::Denied { + denial: None, + failure: Some(MiddlewareFailureDetail { + config_name: None, + code: HttpFailureCode::HookVersionsMixed, + }), + }) + } + ChainHttpHookVersion::Current => { + // A body-aware policy re-checks every replaced body, so the + // upstream must not see any of it before the last stage's output + // is checked. + let delivery = match transformed_body_policy { + openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant => { + delivery + } + openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(_) => { + delivery.or_withhold() + } + }; + Box::pin(apply_pipeline_middleware_chain( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + correlation, + span.clone(), + )) + .await + } } - ChainHttpHookVersion::Current => { - // A body-aware policy re-checks every replaced body, so the - // upstream must not see any of it before the last stage's output - // is checked. - let delivery = match transformed_body_policy { - openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant => { - delivery - } - openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(_) => { - delivery.or_withhold() - } - }; - Box::pin(apply_pipeline_middleware_chain( - req, - client, - ctx, - scheme, - chain, - runner, - generation_guard, - transformed_body_policy, - request_id, - delivery, - )) + }; + if has_stages && let Some(correlation) = correlation { + correlation + .run_stage_classified( + RequestStage::RequestMiddleware, + super::correlation::HttpOutcome::PlatformError, + "request_middleware_failed", + apply, + |outcome| match outcome { + MiddlewareApplyResult::Denied { denial: None, .. } => { + Some("request_middleware_failed") + } + MiddlewareApplyResult::AdmissionExhausted => { + Some("middleware_admission_exhausted") + } + MiddlewareApplyResult::RequestTimeout => Some("request_timeout"), + MiddlewareApplyResult::Allowed(_) + | MiddlewareApplyResult::Streamed { .. } + | MiddlewareApplyResult::Denied { + denial: Some(_), .. + } => None, + }, + ) .await - } + } else { + apply.await } } @@ -916,7 +1126,7 @@ async fn apply_protocol1_middleware_chain Some(admission), openshell_supervisor_middleware::MiddlewareWorkAdmissionOutcome::QueueExhausted => { - emit_middleware_admission_exhausted(ctx); + egress_emit!(middleware_admission_exhausted_event(ctx)); return Ok(MiddlewareApplyResult::AdmissionExhausted); } } @@ -945,7 +1155,7 @@ async fn apply_protocol1_middleware_chain, request_id: &str, delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, + span: tracing::Span, ) -> Result { let head = &req.raw_header[..request_header_end(&req.raw_header)]; // Output that cannot stream to the upstream is withheld up to the route's @@ -1039,6 +1251,7 @@ async fn apply_pipeline_middleware_chain { @@ -1134,6 +1350,7 @@ async fn apply_pipeline_middleware_chain return events.body_failed(error), + (Err(error), _) => return events.body_failed(error, correlation), (Ok(finish), Some(collected)) => (finish, collected), }; generation_guard.ensure_current()?; @@ -1156,7 +1373,13 @@ async fn apply_pipeline_middleware_chain, + span: Option, } impl RequestEvents { @@ -1351,6 +1575,7 @@ impl RequestEvents { reason: &str, denial: Option<&openshell_supervisor_middleware::MiddlewareDenial>, body: HttpStageDiagnostics, + correlation: Option<&super::correlation::RequestCorrelation>, ) { if self.terminal_reason.is_some() { return; @@ -1358,7 +1583,7 @@ impl RequestEvents { self.terminal_reason = Some(reason.to_string()); let mut diagnostics = std::mem::take(&mut self.preflight); diagnostics.extend(body); - for event in request_middleware_events( + let events = request_middleware_events( &self.context, &self.action, &self.target, @@ -1366,13 +1591,17 @@ impl RequestEvents { reason, denial, &diagnostics, - ) { - egress_emit!(event); - } + ); + let span = self.span.take().unwrap_or_else(tracing::Span::none); + span.in_scope(|| emit_request_events(correlation, events)); } /// Record a failed body run and return the result the relay acts on. - fn body_failed(&mut self, error: BodyRunError) -> Result { + fn body_failed( + &mut self, + error: BodyRunError, + correlation: Option<&super::correlation::RequestCorrelation>, + ) -> Result { match error { BodyRunError::Middleware(failure) => { self.emit( @@ -1380,6 +1609,7 @@ impl RequestEvents { &failure.reason, failure.denial.as_ref(), *failure.diagnostics, + correlation, ); Ok(MiddlewareApplyResult::Denied { denial: failure.denial, @@ -1387,16 +1617,20 @@ impl RequestEvents { }) } BodyRunError::ClientTimeout => { - egress_emit!(request_client_timeout_event( - &self.context, - &self.action, - &self.target - )); + emit_request_events( + correlation, + [request_client_timeout_event( + &self.context, + &self.action, + &self.target, + )], + ); self.emit( false, "middleware_cancelled: client_progress_timeout", None, HttpStageDiagnostics::default(), + correlation, ); Ok(MiddlewareApplyResult::RequestTimeout) } @@ -1406,6 +1640,7 @@ impl RequestEvents { "middleware_cancelled: request_body_read_failed", None, HttpStageDiagnostics::default(), + correlation, ); Err(error) } @@ -1461,6 +1696,7 @@ impl RequestBodyStream { client: &mut C, output: mpsc::Sender, upstream_failed: &AtomicBool, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { let session = self .session @@ -1478,11 +1714,17 @@ impl RequestBodyStream { output, openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, ) + .instrument(self.events.span.clone().unwrap_or_else(tracing::Span::none)) .await { Ok(mut finish) => { - self.events - .emit(true, "", None, std::mem::take(&mut finish.diagnostics)); + self.events.emit( + true, + "", + None, + std::mem::take(&mut finish.diagnostics), + correlation, + ); return Ok(finish); } Err(error) => error, @@ -1497,6 +1739,7 @@ impl RequestBodyStream { "middleware_cancelled: upstream_write_failed", None, *failure.diagnostics, + correlation, ); return Err(miette!("upstream stopped accepting the request body")); } @@ -1510,20 +1753,28 @@ impl RequestBodyStream { }, BodyRunError::ClientTimeout => crate::l7::rest::LiveRequestFailure::ClientTimeout, BodyRunError::Client(_) => { - return self.events.body_failed(error).map(|_| unreachable!()); + return self + .events + .body_failed(error, correlation) + .map(|_| unreachable!()); } }; - self.events.body_failed(error)?; + self.events.body_failed(error, correlation)?; Err(miette::Report::new(failure)) } /// Record that the middleware stages were cancelled for `reason`. - pub(crate) fn cancelled(&mut self, reason: &str) { + pub(crate) fn cancelled( + &mut self, + reason: &str, + correlation: Option<&super::correlation::RequestCorrelation>, + ) { self.events.emit( false, &format!("middleware_cancelled: {reason}"), None, HttpStageDiagnostics::default(), + correlation, ); } } @@ -1594,9 +1845,10 @@ fn emit_mixed_protocol_denial( req: &crate::l7::provider::L7Request, chain: &[openshell_supervisor_middleware::DescribedChainEntry], ) { - for event in mixed_protocol_events(ctx, &req.action, &req.target, chain) { - egress_emit!(event); - } + crate::telemetry::emit_decision( + mixed_protocol_events(ctx, &req.action, &req.target, chain), + None, + ); } pub(super) fn mixed_protocol_events( @@ -1649,8 +1901,11 @@ pub async fn send_middleware_rejection_response, failure: Option<&MiddlewareFailureDetail>, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { - let context = Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)); + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); + let context = Some(context); if let Some(denial) = denial { crate::l7::rest::send_middleware_deny_response( req, @@ -1684,13 +1939,16 @@ pub async fn send_middleware_admission_exhausted_response< client: &mut C, ctx: &L7EvalContext, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); crate::l7::rest::send_middleware_unavailable_response( req, &ctx.policy_name, client, Some(redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(context), ) .await?; close_after_early_response(client).await @@ -1703,7 +1961,15 @@ pub async fn send_request_timeout_response( client: &mut C, ctx: &L7EvalContext, redacted_target: &str, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result<()> { + if let Some(correlation) = correlation { + correlation.record_rejection( + 408, + super::correlation::HttpOutcome::PlatformError, + Some("request_timeout"), + ); + } let req = crate::l7::provider::L7Request { action: action.to_string(), target: redacted_target.to_string(), @@ -1711,12 +1977,16 @@ pub async fn send_request_timeout_response( raw_header: Vec::new(), body_length: crate::l7::provider::BodyLength::None, }; + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = correlation.map_or(context, |correlation| { + context.with_request_id(correlation.request_id()) + }); crate::l7::rest::send_request_timeout_response( &req, &ctx.policy_name, client, Some(redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(context), ) .await?; close_after_early_response(client).await @@ -1977,6 +2247,8 @@ pub(super) fn middleware_events( .http_request(HttpRequest::new(&req.action, ctx.ocsf_url(&req.target))) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "request") .unmapped("transformed", invocation.transformed) .unmapped("failed", invocation.failed) .message(format!( @@ -2056,6 +2328,7 @@ pub(super) fn middleware_events( /// Emit the OCSF events describing a middleware chain outcome through the /// tracing pipeline. +#[cfg(test)] fn emit_middleware_events( ctx: &L7EvalContext, req: &crate::l7::provider::L7Request, @@ -2132,6 +2405,7 @@ mod trace_tests { stage_disabled: false, reason_code: None, }], + WebSocketMiddlewarePhase::Message, ); }) .await; @@ -2142,6 +2416,58 @@ mod trace_tests { }) .unwrap(); assert_eq!(finding["action_id"].as_u64(), action_id, "{outcome:?}"); + assert_ne!( + finding["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn websocket_invocation_records_are_explicitly_tagged_by_phase() { + use openshell_supervisor_middleware::{WebSocketInvocation, WebSocketInvocationOutcome}; + + let (events, _) = exported("warn", || { + let invocation = WebSocketInvocation { + config_name: "guard".into(), + implementation: "test/guard".into(), + outcome: WebSocketInvocationOutcome::Allow, + sequence: None, + original_size: 0, + replacement_size: None, + transformed: false, + failed: false, + stage_disabled: false, + reason_code: None, + }; + for phase in [ + WebSocketMiddlewarePhase::Preflight, + WebSocketMiddlewarePhase::SessionStart, + WebSocketMiddlewarePhase::Message, + ] { + emit_websocket_invocations( + &L7EvalContext::default(), + std::slice::from_ref(&invocation), + phase, + ); + } + }) + .await; + + assert_eq!(events.len(), 3); + for (event, phase) in events.iter().zip([ + "websocket_preflight", + "websocket_session_start", + "websocket_message", + ]) { + assert_eq!(event["class_uid"], 4001); + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], phase); + assert_eq!(event["action"], "Allowed"); + assert_eq!(event["status"], "Success"); } } @@ -2163,6 +2489,34 @@ mod trace_tests { } } + #[test] + fn request_invocation_records_are_explicitly_tagged() { + let events = middleware_events( + &L7EvalContext::default(), + &request(), + &outcome( + true, + "", + vec![MiddlewareInvocation { + name: "redact".into(), + implementation: "test/redact".into(), + decision: openshell_core::proto::Decision::Allow, + transformed: true, + failed: false, + }], + vec![], + ), + ); + assert_eq!(events.len(), 1); + let event = events[0].to_json().unwrap(); + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "request"); + assert_eq!(event["unmapped"]["transformed"], true); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn inspection_refusals_record_denial_but_fail_open_findings_do_not() { for denied in [false, true] { @@ -2605,14 +2959,19 @@ mod tests { Some(&denial), None, "/v1/messages", + Some("00000000-0000-4000-8000-000000000010"), ), read ); sent.expect("send denial"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000010\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_denied"); + assert!(body.get("request_id").is_none()); assert_eq!(body["middleware"], "prototype-content-guard"); assert_eq!(body["reason_code"], "content_match"); assert!(body.get("rule_missing").is_none()); @@ -2713,14 +3072,19 @@ mod tests { None, Some(&failure), "/v1/messages", + Some("00000000-0000-4000-8000-000000000011"), ), read ); sent.expect("send failure"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000011\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_failed"); + assert!(body.get("request_id").is_none()); assert_eq!( body["detail"], "Request could not be processed by configured middleware" @@ -2765,7 +3129,15 @@ mod tests { response }; let (sent, response) = tokio::join!( - send_middleware_rejection_response(&req, &mut server, &ctx, None, None, "/upload"), + send_middleware_rejection_response( + &req, + &mut server, + &ctx, + None, + None, + "/upload", + None + ), upload ); sent.expect("send failure"); diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index c24777826e..0647b8b2fe 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -8,6 +8,7 @@ //! doing a raw `copy_bidirectional`. Each request within the tunnel is parsed, //! evaluated against OPA policy, and either forwarded or denied. +pub(crate) mod correlation; pub mod graphql; pub(crate) mod http; pub mod jsonrpc; @@ -16,6 +17,7 @@ pub(crate) mod middleware; pub mod path; pub mod provider; pub mod relay; +mod request_trace; pub mod rest; pub mod tls; pub(crate) mod token_grant_injection; diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 361895db5e..9725ef0ea5 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,13 +7,16 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. +use crate::l7::correlation::{DetachedRequestEvent, HttpOutcome, RequestCorrelation, RequestStage}; use crate::l7::middleware::{ MiddlewareApplyResult, RequestBodyDelivery, UninspectableTrafficGate, - apply_middleware_chain_with_request_id, apply_middleware_chain_with_request_id_and_delivery, - emit_middleware_uninspectable, middleware_network_input, uninspectable_traffic_gate, + apply_middleware_chain_with_request_id_and_delivery_correlated, + apply_middleware_chain_with_request_id_correlated, emit_middleware_uninspectable, + middleware_network_input, uninspectable_traffic_gate, }; #[cfg(test)] use crate::l7::middleware::{ + apply_middleware_chain_with_request_id, apply_middleware_chain_with_request_id_and_delivery, middleware_chain_body_limit, middleware_events, middleware_request_input, raw_query_from_request_headers, resolve_unbuffered_body, }; @@ -37,6 +40,8 @@ use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; +#[cfg(test)] +pub(crate) mod correlation_dispatch_tests; #[cfg(test)] mod token_grant_ownership_tests; @@ -163,6 +168,7 @@ fn credential_generation_guard( /// /// A refused grant returns a [`TokenGrantInjectionError`](crate::l7::token_grant_injection::TokenGrantInjectionError) /// inside the report, so callers can tell it apart from other failures. +#[allow(clippy::too_many_arguments)] pub(crate) async fn prepare_inspected_request( req: crate::l7::provider::L7Request, ctx: &L7EvalContext, @@ -171,6 +177,7 @@ pub(crate) async fn prepare_inspected_request( request_info: &L7RequestInfo, request_id: &str, observation_context: Option<&openshell_core::endpoint_status::EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<( crate::l7::provider::L7Request, L7EvalContext, @@ -217,7 +224,7 @@ pub(crate) async fn prepare_inspected_request( L7Protocol::Rest | L7Protocol::Websocket | L7Protocol::Sql => {} } let owners = admitted_token_grant_owners(engine, &scoped, ¤t_info)?; - crate::l7::token_grant_injection::inject_for_admitted_owners( + crate::l7::token_grant_injection::inject_for_admitted_owners_correlated( req, &scoped, snapshot, @@ -226,6 +233,7 @@ pub(crate) async fn prepare_inspected_request( observation_context.filter(|context| { context.matches_config(&config.policy_hash, Some(snapshot.revision)) }), + correlation, ) .await? } else { @@ -280,35 +288,48 @@ async fn reject_request_authority_mismatch( client: &mut W, ctx: &L7EvalContext, method: &str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, { - let body = r#"{"error":"request_authority_mismatch","message":"HTTP request authority does not match the authorized tunnel endpoint"}"#; - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - - crate::telemetry::emit_decision( + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + ) + .await?; + } + crate::l7::correlation::emit_request_events( + correlation, [ build_request_authority_mismatch_event(ctx, method), build_request_authority_mismatch_finding(ctx), ], - None, ); Ok(()) } -/// Return the selected revision's inspection for policy evaluation, or emit a -/// rejection and return `None`. Non-MCP adapters retain their original inspection. -/// Record local rejections before client response delivery can fail. -pub(crate) async fn enforce_mcp_protocol_version( +#[allow(clippy::too_many_arguments)] +pub(crate) async fn enforce_mcp_protocol_version_correlated( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, mut info: crate::l7::jsonrpc::JsonRpcRequestInfo, @@ -316,6 +337,7 @@ pub(crate) async fn enforce_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result> where W: AsyncWrite + Unpin, @@ -351,7 +373,7 @@ where crate::l7::mcp::selected_revision_context(request, version) ); let summary = l7_protocol_log_summary(None, Some(&info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -359,7 +381,13 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, true, "l7_parse_rejection"); let body = serde_json::json!({ "error": "invalid_mcp_request", @@ -370,13 +398,24 @@ where "method": request.action, "path": redacted_target, }); - crate::l7::rest::send_json_response( - &ctx.policy_name, - body, - client, - "400 Bad Request", - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "400 Bad Request", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response( + &ctx.policy_name, + body, + client, + "400 Bad Request", + ) + .await?; + } return Ok(None); } @@ -389,6 +428,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; return Ok(None); @@ -405,6 +445,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; Ok(None) @@ -413,6 +454,7 @@ where } /// Emit the same transport or policy rejection at initial and final inspection. +#[allow(clippy::too_many_arguments)] async fn reject_mcp_protocol_version( error: crate::l7::mcp::McpProtocolVersionError, request: &crate::l7::provider::L7Request, @@ -421,6 +463,7 @@ async fn reject_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -432,7 +475,7 @@ where } let reason = error.rejection_detail(request); let summary = l7_protocol_log_summary(None, Some(info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -440,7 +483,20 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + let status = match error { + crate::l7::mcp::McpProtocolVersionError::InvalidHeader + | crate::l7::mcp::McpProtocolVersionError::UnsupportedHeaderValue + | crate::l7::mcp::McpProtocolVersionError::InvalidRequestMetadata => 400, + crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed => 405, + crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => 403, + }; + correlation.record_rejection(status, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + egress_emit!(event); + } let deny_group = match error { crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => "l7_policy", crate::l7::mcp::McpProtocolVersionError::InvalidHeader @@ -460,19 +516,32 @@ where }); let allowed_methods = (error == crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed).then_some("POST"); - crate::l7::rest::send_json_response_with_allow( - &ctx.policy_name, - body, - client, - error.http_status(), - allowed_methods, - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_allow_and_request_id( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response_with_allow( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + ) + .await?; + } Ok(()) } /// Reinspect the buffered outgoing MCP request after request transformations. /// The forwarding adapter must call this before any upstream request write. +#[allow(clippy::too_many_arguments)] pub(crate) async fn enforce_final_mcp_protocol_version( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, @@ -480,6 +549,7 @@ pub(crate) async fn enforce_final_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result where W: AsyncWrite + Unpin, @@ -491,7 +561,7 @@ where request, crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), )?; - Ok(enforce_mcp_protocol_version( + Ok(enforce_mcp_protocol_version_correlated( config, request, info, @@ -499,6 +569,7 @@ where ctx, redacted_target, observer, + correlation, ) .await? .is_some()) @@ -601,6 +672,7 @@ pub(crate) async fn reject_credential_resolution( ctx: &L7EvalContext, method: &str, error: &secrets::UnresolvedPlaceholderError, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -613,22 +685,43 @@ where "500 Internal Server Error" }; let body = if placement_mismatch { - r#"{"error":"credential_placement_mismatch","message":"Credential is not authorized for this request location"}"# + serde_json::json!({ + "error": "credential_placement_mismatch", + "message": "Credential is not authorized for this request location", + }) } else if endpoint_mismatch { - r#"{"error":"credential_endpoint_mismatch","message":"Credential is not authorized for this request endpoint"}"# + serde_json::json!({ + "error": "credential_endpoint_mismatch", + "message": "Credential is not authorized for this request endpoint", + }) } else { - r#"{"error":"credential_unavailable","message":"Credential placeholder could not be resolved"}"# + serde_json::json!({ + "error": "credential_unavailable", + "message": "Credential placeholder could not be resolved", + }) }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - + if let Some(correlation) = correlation { + let (response_code, outcome) = if endpoint_mismatch || placement_mismatch { + (403, HttpOutcome::Denied) + } else { + (500, HttpOutcome::PlatformError) + }; + correlation.record_rejection( + response_code, + outcome, + (!endpoint_mismatch && !placement_mismatch).then_some("credential_resolution_failed"), + ); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + status, + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, status).await?; + } let mut events = if placement_mismatch { vec![ HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -660,28 +753,34 @@ where if endpoint_mismatch { events.push(build_credential_endpoint_mismatch_finding(ctx)); } - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(correlation, events); Ok(()) } pub(crate) async fn reject_body_credential( client: &mut C, error: secrets::body::BodyCredentialError, + ctx: &L7EvalContext, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { let body = serde_json::json!({"error": { "code": "credential_placeholder_in_request_body", "reason": error.reason(), "message": "A credential placeholder in the request body cannot be forwarded. Remove the reference from conversation history or restore provider access; body credential rewriting is disabled." - }}).to_string(); - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) + }}); + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) .await - .into_diagnostic()?; - client.flush().await.into_diagnostic() + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden").await + } } struct InspectedForwarding<'a> { @@ -689,6 +788,7 @@ struct InspectedForwarding<'a> { engine: &'a TunnelPolicyEngine, request_info: &'a L7RequestInfo, request_id: &'a str, + correlation: Option<&'a RequestCorrelation>, response_chain: &'a [openshell_supervisor_middleware::ChainEntry], websocket_middleware: bool, observation_context: Option<&'a openshell_core::endpoint_status::EndpointObservationContext>, @@ -715,6 +815,7 @@ where engine, request_info, request_id, + correlation, response_chain, websocket_middleware, observation_context, @@ -732,11 +833,19 @@ where request_info, request_id, observation_context, + correlation, ) .await { Ok(prepared) => prepared, Err(error) => { + if let Some(correlation) = correlation { + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("credential_prepare_failed"), + ); + } if let Some(refusal) = error.downcast_ref::() { @@ -755,7 +864,8 @@ where write_bad_gateway_response(client, Some(request_id)).await?; } else { warn!(error = %error, "Token grant failed before forwarding"); - write_bad_gateway_response(client, None).await?; + write_bad_gateway_response(client, correlation.map(RequestCorrelation::request_id)) + .await?; } return Ok(None); } @@ -813,7 +923,10 @@ where engine.middleware_runner(), Some(engine.generation_guard()), )), - observer.as_ref(), + RelayHttpInstrumentation { + observer: observer.as_ref(), + correlation, + }, )) .await } @@ -838,11 +951,17 @@ where options, ctx, response_middleware, - None, + RelayHttpInstrumentation::default(), ) .await } +#[derive(Clone, Copy, Default)] +struct RelayHttpInstrumentation<'a> { + observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, +} + #[cfg(test)] async fn relay_http_request_with_credential_rejection_observed( request: &crate::l7::provider::L7Request, @@ -851,7 +970,7 @@ async fn relay_http_request_with_credential_rejection_observed( options: crate::l7::rest::RelayRequestOptions<'_>, ctx: &L7EvalContext, response_middleware: Option>, - observer: Option<&EndpointObserver>, + instrumentation: RelayHttpInstrumentation<'_>, ) -> Result> where C: AsyncRead + AsyncWrite + Unpin, @@ -865,7 +984,7 @@ where ctx, None, response_middleware, - observer, + instrumentation, )) .await } @@ -882,14 +1001,18 @@ async fn relay_http_request_with_body_observed( ctx: &L7EvalContext, live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, response_middleware: Option>, - observer: Option<&EndpointObserver>, + instrumentation: RelayHttpInstrumentation<'_>, ) -> Result> where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { + let RelayHttpInstrumentation { + observer, + correlation, + } = instrumentation; match Box::pin( - crate::l7::rest::relay_http_request_with_body_guarded_observed( + crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( request, client, upstream, @@ -897,6 +1020,7 @@ where live_body, response_middleware, observer, + correlation, ), ) .await @@ -905,7 +1029,8 @@ where Err(report) => { if let Some(failure) = report.downcast_ref::() { let _ = upstream.shutdown().await; - reject_live_request_body(client, ctx, request, failure, observer).await?; + reject_live_request_body(client, ctx, request, failure, observer, correlation) + .await?; return Ok(None); } if let Some(error) = report.downcast_ref::() { @@ -916,14 +1041,15 @@ where observer.observe(EndpointResult::PolicyDenied); } let _ = upstream.shutdown().await; - reject_body_credential(client, *error).await?; + reject_body_credential(client, *error, ctx, correlation).await?; Ok(None) } else if let Some(error) = report.downcast_ref::() { if let Some(observer) = observer { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &request.action, error).await?; + reject_credential_resolution(client, ctx, &request.action, error, correlation) + .await?; Ok(None) } else if report .downcast_ref::() @@ -932,6 +1058,10 @@ where if let Some(observer) = observer { observer.observe_credential_failure(false); } + if let Some(correlation) = correlation { + correlation + .record_failure(HttpOutcome::PlatformError, "credential_unavailable"); + } Err(report) } else { Err(report) @@ -948,6 +1078,7 @@ async fn reject_live_request_body( request: &crate::l7::provider::L7Request, failure: &crate::l7::rest::LiveRequestFailure, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { let redacted_target = secrets::redact_target_for_policy(&request.target) .unwrap_or_else(|_| "[REDACTED]".to_string()); @@ -958,11 +1089,25 @@ async fn reject_live_request_body( raw_header: Vec::new(), body_length: crate::l7::provider::BodyLength::None, }; - let context = Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)); + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = Some(correlation.map_or(context, |correlation| { + context.with_request_id(correlation.request_id()) + })); match failure { crate::l7::rest::LiveRequestFailure::Middleware { denial, failure, .. } => { + if let Some(correlation) = correlation { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); + } if let Some(observer) = observer { observer.observe(EndpointResult::PolicyDenied); } @@ -989,6 +1134,13 @@ async fn reject_live_request_body( } } crate::l7::rest::LiveRequestFailure::ClientTimeout => { + if let Some(correlation) = correlation { + correlation.record_rejection( + 408, + HttpOutcome::PlatformError, + Some("request_timeout"), + ); + } crate::l7::rest::send_request_timeout_response( &rejected, &ctx.policy_name, @@ -1120,6 +1272,7 @@ pub(crate) struct UpgradeRelayOptions<'a> { pub(crate) policy_name: String, pub(crate) middleware_session: Option, pub(crate) selected_subprotocol: Option, + pub(crate) handshake_event: Option, } #[derive(Default)] @@ -1218,6 +1371,7 @@ async fn deny_unsupported_upgrade_if_requested( ctx: &L7EvalContext, observer: Option<&EndpointObserver>, client: &mut C, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin + Send, @@ -1231,17 +1385,27 @@ where if let Some(observer) = observer { observer.observe(EndpointResult::PolicyDenied); } + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + } emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); - crate::l7::rest::send_json_response( - &ctx.policy_name, - serde_json::json!({ - "error": "unsupported_l7_protocol", - "detail": detail, - }), - client, - "403 Forbidden", - ) - .await?; + let body = serde_json::json!({ + "error": "unsupported_l7_protocol", + "detail": detail, + }); + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden") + .await?; + } Ok(true) } @@ -1329,6 +1493,23 @@ pub async fn relay_with_route_selection( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_with_route_selection_inner( + configs, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_with_route_selection_inner( + configs: &[L7EndpointConfig], + engine: TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -1378,20 +1559,27 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } let route_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } }; let Some(config) = select_l7_config_for_path(configs, &route_target) else { let reason = "no L7 endpoint path matched request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); emit_l7_request_log(ctx, &req.action, &route_target, "deny", "l7", reason, None); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( @@ -1400,9 +1588,13 @@ where reason, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); }; let observer = EndpointObserver::begin_captured( @@ -1429,6 +1621,9 @@ where observer.observe(EndpointResult::PolicyDenied); } emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); + correlation.record_rejection(403, HttpOutcome::Denied, None); + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1436,13 +1631,22 @@ where detail, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await + .map_err(&capture_error)? { return Ok(()); } @@ -1467,6 +1671,9 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + { + correlation.record_outcome(HttpOutcome::Denied); + } emit_parse_rejection(ctx, &detail, "l7-graphql"); } return Ok(()); @@ -1503,6 +1710,9 @@ where &e.to_string(), ParseRejectionMode::L7Endpoint, ); + { + correlation.record_outcome(HttpOutcome::Denied); + } emit_parse_rejection(ctx, &detail, "l7-jsonrpc"); } return Ok(()); @@ -1523,13 +1733,15 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } }; if let Some(info) = jsonrpc_info.take() { - let Some(inspected) = enforce_mcp_protocol_version( + let Some(inspected) = enforce_mcp_protocol_version_correlated( config, &req, info, @@ -1537,8 +1749,10 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -1553,16 +1767,21 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } @@ -1571,7 +1790,14 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(&engine, ctx, &request_info)? + correlation + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(&engine, ctx, &request_info), + ) + .map_err(&capture_error)? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -1598,15 +1824,18 @@ where ); if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // Route selection resolved `config` per request, so re-check the // body against that protocol's policy after every transforming // stage (a no-op for REST and websocket, whose policy inputs the // chain cannot mutate). - let validate = transformed_body_validator(config, &engine, ctx, &request_info); + let validate = + transformed_body_validator(config, &engine, ctx, &request_info, Some(&correlation)); let transformed_body_policy = if matches!( config.protocol, L7Protocol::Rest | L7Protocol::Websocket | L7Protocol::Sql @@ -1615,7 +1844,7 @@ where } else { openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate) }; - let middleware_result = apply_middleware_chain_with_request_id_and_delivery( + let middleware_result = apply_middleware_chain_with_request_id_and_delivery_correlated( req, client, ctx, @@ -1623,11 +1852,12 @@ where engine.middleware_runner(), engine.generation_guard(), transformed_body_policy, - &request_id, + request_id, request_body_delivery(config), + Some(&correlation), ) .await; - let middleware_result = middleware_result?; + let middleware_result = middleware_result.map_err(&capture_error)?; let (req, mut live_body) = match middleware_result { MiddlewareApplyResult::Allowed(request) => (request, None), MiddlewareApplyResult::Streamed { request, body } => (request, Some(body)), @@ -1637,14 +1867,25 @@ where client, ctx, &redacted_target, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1659,11 +1900,18 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1676,8 +1924,10 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -1688,18 +1938,48 @@ where engine.middleware_runner(), ctx, ctx.websocket_scheme(), - ) - .await; + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); + write_bad_gateway_response(client, Some(correlation.request_id())) + .await + .map_err(&capture_error)?; return Ok(()); } }; crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + preflight + .denial + .is_none() + .then_some("websocket_preflight_failed"), + ); crate::l7::middleware::send_middleware_rejection_response( &req, client, @@ -1707,8 +1987,10 @@ where preflight.denial.as_ref(), None, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } preflight.session @@ -1726,7 +2008,8 @@ where config, engine: &engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -1755,7 +2038,10 @@ where &ctx.policy_name, outcome_result, ) - .await?; + .await + .map_err(&capture_error)?; + drop(live_body); + drop(validate); match outcome { RelayOutcome::Reusable => { if let Some(session) = middleware_session.take() { @@ -1809,6 +2095,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; + options.handshake_event = correlation.finish_http_request_for_event(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -1819,6 +2106,12 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + { + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); + } + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1826,9 +2119,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -2032,18 +2326,21 @@ where } else { "raw bidirectional relay (L7 enforcement no longer active)" }; - egress_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .activity_name("Upgrade") - .severity(SeverityId::Informational) - .dst_endpoint(Endpoint::from_domain(host, port)) - .message(format!( - "101 Switching Protocols — {relay_mode} [host:{host} port:{port} overflow_bytes:{}]", - overflow.len() - )) - .build() - ); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .activity_name("Upgrade") + .severity(SeverityId::Informational) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!( + "101 Switching Protocols — {relay_mode} [host:{host} port:{port} overflow_bytes:{}]", + overflow.len() + )) + .build(); + if let Some(handshake_event) = options.handshake_event.take() { + handshake_event.emit(event); + } else { + egress_emit!(event); + } if use_websocket_relay { let assembly_budget = options.assembly_budget.take().unwrap_or_default(); let resolver = if options.websocket.credential_rewrite { @@ -2196,6 +2493,7 @@ pub(crate) fn upgrade_options<'a>( policy_name: ctx.policy_name.clone(), middleware_session: None, selected_subprotocol: None, + handshake_event: None, } } @@ -2228,6 +2526,23 @@ async fn relay_rest( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_rest_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_rest_inner( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -2270,11 +2585,25 @@ where return Ok(()); // Close connection on parse error } }; + let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await + .map_err(&capture_error)? + { return Ok(()); } @@ -2287,7 +2616,9 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -2301,21 +2632,34 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); provider .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } // Evaluate L7 policy via Rego (using redacted target) - let (allowed, reason) = evaluate_l7_request(engine, ctx, &request_info)?; + let (allowed, reason) = correlation + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(engine, ctx, &request_info), + ) + .map_err(&capture_error)?; if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); @@ -2373,18 +2717,20 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || config.enforcement == EnforcementMode::Audit { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // REST and websocket-upgrade policy evaluates only the method, // path, and query, which a middleware result cannot mutate, so no // per-stage body re-check is needed. - let middleware_result = apply_middleware_chain_with_request_id_and_delivery( + let middleware_result = apply_middleware_chain_with_request_id_and_delivery_correlated( req, client, ctx, @@ -2392,11 +2738,12 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, - &request_id, + request_id, request_body_delivery(config), + Some(&correlation), ) .await; - let middleware_result = middleware_result?; + let middleware_result = middleware_result.map_err(&capture_error)?; let (req, mut live_body) = match middleware_result { MiddlewareApplyResult::Allowed(request) => (request, None), MiddlewareApplyResult::Streamed { request, body } => (request, Some(body)), @@ -2406,11 +2753,22 @@ where client, ctx, &redacted_target, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2425,11 +2783,18 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2442,8 +2807,10 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -2454,18 +2821,48 @@ where engine.middleware_runner(), ctx, ctx.websocket_scheme(), - ) - .await; + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); + write_bad_gateway_response(client, Some(correlation.request_id())) + .await + .map_err(&capture_error)?; return Ok(()); } }; crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + preflight + .denial + .is_none() + .then_some("websocket_preflight_failed"), + ); crate::l7::middleware::send_middleware_rejection_response( &req, client, @@ -2473,8 +2870,10 @@ where preflight.denial.as_ref(), None, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } preflight.session @@ -2492,7 +2891,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -2521,7 +2921,8 @@ where &ctx.policy_name, outcome_result, ) - .await?; + .await + .map_err(&capture_error)?; match outcome { RelayOutcome::Reusable => { if let Some(session) = middleware_session.take() { @@ -2565,6 +2966,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; + options.handshake_event = correlation.finish_http_request_for_event(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -2573,6 +2975,8 @@ where } } else { // Enforce mode: deny with 403 and close connection (use redacted target) + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); provider .deny_with_redacted_target( &req, @@ -2580,9 +2984,13 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -2603,6 +3011,7 @@ pub(crate) fn emit_policy_reload( port: u16, policy_name: &str, ) { + crate::l7::correlation::record_current_failure(HttpOutcome::PlatformError, "policy_reloaded"); egress_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) @@ -2661,7 +3070,24 @@ pub(crate) async fn finalize_websocket_pre_upgrade( } } -async fn relay_jsonrpc( +async fn relay_jsonrpc( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_jsonrpc_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_jsonrpc_inner( config: &L7EndpointConfig, engine: &TunnelPolicyEngine, client: &mut C, @@ -2715,6 +3141,8 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; + let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), config, @@ -2726,11 +3154,21 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await + .map_err(&capture_error)? { return Ok(()); } @@ -2744,12 +3182,14 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } }; - let Some(jsonrpc_info) = enforce_mcp_protocol_version( + let Some(jsonrpc_info) = enforce_mcp_protocol_version_correlated( config, &req, jsonrpc_info, @@ -2757,8 +3197,10 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -2776,8 +3218,21 @@ where let (allowed, reason, jsonrpc_log_info) = if let Some(reason) = hard_deny_reason { (false, reason, jsonrpc_info.clone()) } else { - let evaluation = - evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info)?; + let evaluation = correlation + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || { + evaluate_jsonrpc_l7_request_for_log( + engine, + ctx, + &request_info, + &jsonrpc_info, + ) + }, + ) + .map_err(&capture_error)?; (evaluation.allowed, evaluation.reason, evaluation.log_info) }; @@ -2823,19 +3278,22 @@ where &reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the // upstream or the next stage. - let validate = transformed_body_validator(config, engine, ctx, &request_info); - let middleware_result = apply_middleware_chain_with_request_id( + let validate = + transformed_body_validator(config, engine, ctx, &request_info, Some(&correlation)); + let middleware_result = apply_middleware_chain_with_request_id_correlated( req, client, ctx, @@ -2843,15 +3301,17 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; let req = match middleware_result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { .. } => { - return Err(miette!( + return Err(correlation.capture_error(miette!( "body-aware request middleware must collect the request body" - )); + ))); } // The body was read before middleware ran, so the client // cannot stall it; answer like the REST path if it happens. @@ -2861,14 +3321,25 @@ where client, ctx, &redacted_target, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2883,11 +3354,18 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2900,8 +3378,10 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -2915,14 +3395,16 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), live_body: None, }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -2944,6 +3426,8 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -2951,9 +3435,13 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -2966,6 +3454,23 @@ async fn relay_graphql( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_graphql_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_graphql_inner( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -3008,12 +3513,26 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await + .map_err(&capture_error)? + { return Ok(()); } @@ -3036,6 +3555,7 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + correlation.record_outcome(HttpOutcome::Denied); emit_parse_rejection(ctx, &detail, "l7-graphql"); } return Ok(()); @@ -3046,7 +3566,9 @@ where // without Host must still be denied if that body contains reserved // credential markers, so repeat the authority check on the full request. if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } @@ -3057,7 +3579,9 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3079,7 +3603,14 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(engine, ctx, &request_info)? + correlation + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(engine, ctx, &request_info), + ) + .map_err(&capture_error)? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -3124,19 +3655,22 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the // upstream or the next stage. - let validate = transformed_body_validator(config, engine, ctx, &request_info); - let middleware_result = apply_middleware_chain_with_request_id( + let validate = + transformed_body_validator(config, engine, ctx, &request_info, Some(&correlation)); + let middleware_result = apply_middleware_chain_with_request_id_correlated( req, client, ctx, @@ -3144,15 +3678,17 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; let req = match middleware_result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { .. } => { - return Err(miette!( + return Err(correlation.capture_error(miette!( "body-aware request middleware must collect the request body" - )); + ))); } // The body was read before middleware ran, so the client // cannot stall it; answer like the REST path if it happens. @@ -3162,11 +3698,22 @@ where client, ctx, &redacted_target, + Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3181,11 +3728,18 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3198,8 +3752,10 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3213,14 +3769,16 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), live_body: None, }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -3251,6 +3809,8 @@ where } } } else { + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -3258,9 +3818,13 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -3504,6 +4068,7 @@ fn reevaluate_transformed_body( ctx: &L7EvalContext, request_info: &L7RequestInfo, body: &[u8], + correlation: Option<&RequestCorrelation>, ) -> Result> { let (engine_type, transformed_info) = match config.protocol { // REST and websocket-upgrade policy evaluates only the method, path, @@ -3551,20 +4116,42 @@ fn reevaluate_transformed_body( if let Some(reason) = l7_request_hard_deny_reason(config.protocol, &transformed_info) { let reason = format!("middleware transformation rejected: {reason}"); - emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + emit_transformed_body_decision( + ctx, + request_info, + engine_type, + "deny", + &reason, + correlation, + ); return Ok(Some(reason)); } - let (allowed, reason) = evaluate_l7_request(engine, ctx, &transformed_info)?; + let evaluate = || evaluate_l7_request(engine, ctx, &transformed_info); + let (allowed, reason) = correlation.map_or_else(evaluate, |correlation| { + correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + evaluate, + ) + })?; if allowed { return Ok(None); } let reason = format!("middleware transformation denied by policy: {reason}"); if config.enforcement == EnforcementMode::Audit { - emit_transformed_body_decision(ctx, request_info, engine_type, "audit", &reason); + emit_transformed_body_decision( + ctx, + request_info, + engine_type, + "audit", + &reason, + correlation, + ); return Ok(None); } - emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason, correlation); Ok(Some(reason)) } @@ -3576,8 +4163,11 @@ pub(crate) fn transformed_body_validator<'a>( engine: &'a TunnelPolicyEngine, ctx: &'a L7EvalContext, request_info: &'a L7RequestInfo, + correlation: Option<&'a RequestCorrelation>, ) -> impl Fn(&[u8]) -> Result> + Send + Sync + 'a { - move |body: &[u8]| reevaluate_transformed_body(config, engine, ctx, request_info, body) + move |body: &[u8]| { + reevaluate_transformed_body(config, engine, ctx, request_info, body, correlation) + } } /// Log the post-transformation policy decision as an OCSF HTTP Activity @@ -3589,6 +4179,7 @@ fn emit_transformed_body_decision( engine_type: &str, decision_str: &str, reason: &str, + correlation: Option<&RequestCorrelation>, ) { let (action_id, disposition_id, severity) = match decision_str { "deny" => (ActionId::Denied, DispositionId::Blocked, SeverityId::Medium), @@ -3614,7 +4205,7 @@ fn emit_transformed_body_decision( request_info.action, ctx.host, ctx.port, request_info.target, reason )) .build(); - egress_emit!(event); + crate::l7::correlation::emit_request_events(correlation, [event]); } fn jsonrpc_policy_input(info: &crate::l7::jsonrpc::JsonRpcRequestInfo) -> serde_json::Value { @@ -3774,6 +4365,27 @@ pub async fn relay_passthrough_with_credentials( generation_guard: &PolicyGenerationGuard, middleware_engine: Option<&crate::opa::OpaEngine>, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_passthrough_with_credentials_inner( + client, + upstream, + ctx, + generation_guard, + middleware_engine, + )) + .await +} + +async fn relay_passthrough_with_credentials_inner( + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, + generation_guard: &PolicyGenerationGuard, + middleware_engine: Option<&crate::opa::OpaEngine>, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -3795,8 +4407,20 @@ where .as_ref() .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); + let middleware_selection = middleware_engine.map(|engine| { + engine.query_middleware_chain_with_generation(&middleware_network_input(ctx)) + }); + let inspected = matches!( + &middleware_selection, + Some(Ok((chain, _))) if !chain.is_empty() + ); // Read next request from client. - let req = match provider.parse_request(client).await { + let parsed = if inspected { + provider.parse_request(client).await + } else { + provider.parse_uninspected_request(client).await + }; + let req = match parsed { Ok(Some(req)) => req, Ok(None) => break, // Client closed connection. Err(e) => { @@ -3809,8 +4433,17 @@ where return Ok(()); } }; + let correlation = if inspected { + RequestCorrelation::new(&req, ctx) + } else { + RequestCorrelation::uninspected() + }; + let capture_error = |error| correlation.capture_error(error); + let receipt = inspected.then_some(&correlation); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, receipt) + .await + .map_err(&capture_error)?; return Ok(()); } if close_if_stale(generation_guard, ctx) { @@ -3823,7 +4456,9 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, receipt) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3847,19 +4482,18 @@ where req.action, ctx.host, ctx.port, redacted_target, )) .build(); - egress_emit!(event); + correlation.emit(event); } - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id().to_string(); let mut response_selection = None; let mut live_body = None; - let req = if let Some(engine) = middleware_engine { - let input = middleware_network_input(ctx); - let (chain, generation) = engine.query_middleware_chain_with_generation(&input)?; + let req = if let Some((engine, selection)) = middleware_engine.zip(middleware_selection) { + let (chain, generation) = selection.map_err(&capture_error)?; if generation != generation_guard.captured_generation() { return Ok(()); } - let runner = engine.middleware_runner()?; + let runner = engine.middleware_runner().map_err(&capture_error)?; let exchange = crate::l7::middleware::HttpMiddlewareExchange::new( request_id.clone(), chain, @@ -3869,14 +4503,16 @@ where // The passthrough path enforces no L7 policy, so there is no // body-aware decision to re-check after a transformation. let result = exchange - .apply_request( + .apply_request_correlated( req, client, ctx, ctx.request_scheme(), openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, + receipt, ) - .await?; + .await + .map_err(&capture_error)?; let request = match result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { request, body } => { @@ -3889,11 +4525,22 @@ where client, ctx, &redacted_target, + receipt, ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let denied_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -3908,11 +4555,18 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + receipt.map(RequestCorrelation::request_id), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -3925,8 +4579,10 @@ where client, ctx, &redacted_target, + receipt.map(RequestCorrelation::request_id), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3939,12 +4595,13 @@ where let injected = match crate::l7::token_grant_injection::pin_live_credentials(&mut grant_ctx) { Some(snapshot) => { - crate::l7::token_grant_injection::inject_for_selectors( + crate::l7::token_grant_injection::inject_for_selectors_correlated( req, &grant_ctx, &snapshot, &request_id, observation_context.as_ref(), + receipt, ) .await } @@ -3954,9 +4611,16 @@ where )), }; let Ok((req_with_auth, injected)) = injected else { + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("credential_prepare_failed"), + ); // The refusal event names the credential and request ID. // The sandbox-level grant record does not require an MCP endpoint identity. - write_bad_gateway_response(client, Some(&request_id)).await?; + write_bad_gateway_response(client, Some(&request_id)) + .await + .map_err(&capture_error)?; return Ok(()); }; let ctx = &grant_ctx; @@ -3986,9 +4650,13 @@ where ctx, live_body.as_deref_mut(), response_middleware, - None, + RelayHttpInstrumentation { + observer: None, + correlation: receipt, + }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -4003,7 +4671,10 @@ where overflow, &ctx.host, ctx.port, - UpgradeRelayOptions::default(), + UpgradeRelayOptions { + handshake_event: correlation.finish_http_request_for_event(), + ..Default::default() + }, ) .await; } @@ -4057,12 +4728,12 @@ mod trace_tests { ..Default::default() }; let mut response = Vec::new(); - // The in-memory span processor exports with a futures executor. tokio::task::block_in_place(|| { tokio::runtime::Handle::current().block_on(reject_request_authority_mismatch( &mut response, &ctx, "POST", + None, )) }) .unwrap(); @@ -4108,6 +4779,7 @@ mod trace_tests { mod tests { use super::*; use crate::opa::{NetworkInput, OpaEngine}; + use crate::test_support::response_request_id; use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::token_grant_failure::{ @@ -4118,6 +4790,20 @@ mod tests { use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; + fn assert_http_control_records( + events: &[serde_json::Value], + expected_controls: usize, + expected_invocations: usize, + ) -> (Vec<&serde_json::Value>, Vec<&serde_json::Value>) { + let (invocations, controls): (Vec<_>, Vec<_>) = events + .iter() + .filter(|event| event["class_uid"] == 4002) + .partition(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation"); + assert_eq!(controls.len(), expected_controls, "{events:?}"); + assert_eq!(invocations.len(), expected_invocations, "{events:?}"); + (controls, invocations) + } + fn relay_grant_failure() -> TokenGrantFailure { TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, @@ -4143,6 +4829,7 @@ mod tests { #[tokio::test] async fn body_denial_returns_actionable_local_json() { + const REQUEST_ID: &str = "8b02c179-9e6b-4eb0-948b-d9f40582fc44"; let req = crate::l7::provider::L7Request { action: "POST".into(), target: "/v1/responses".into(), query_params: TestHashMap::new(), raw_header: b"POST /v1/responses HTTP/1.1\r\nHost: api.openai.com\r\nContent-Length: 25\r\n\r\nopenshell:resolve:env:KEY".to_vec(), @@ -4150,7 +4837,12 @@ mod tests { }; let (mut client, mut caller) = tokio::io::duplex(4096); let (mut upstream, mut server) = tokio::io::duplex(4096); - let outcome = relay_http_request_with_credential_rejection( + let ctx = L7EvalContext { + policy_name: "body-policy".into(), + ..Default::default() + }; + let correlation = RequestCorrelation::with_request_id(&req, &ctx, REQUEST_ID.into()); + let outcome = relay_http_request_with_credential_rejection_observed( &req, &mut client, &mut upstream, @@ -4158,8 +4850,12 @@ mod tests { deny_uninspected_credentials: true, ..Default::default() }, - &L7EvalContext::default(), + &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .unwrap(); @@ -4168,6 +4864,10 @@ mod tests { let mut response = String::new(); caller.read_to_string(&mut response).await.unwrap(); assert!(response.starts_with("HTTP/1.1 403 Forbidden")); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); let body: serde_json::Value = serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); assert_eq!( @@ -4175,6 +4875,7 @@ mod tests { "credential_placeholder_in_request_body" ); assert_eq!(body["error"]["reason"], "classification_unavailable"); + assert!(body.get("request_id").is_none()); let mut sent = String::new(); server.read_to_string(&mut sent).await.unwrap(); assert!(!sent.contains("openshell:resolve:")); @@ -4248,7 +4949,10 @@ mod tests { }, &L7EvalContext::default(), None, - Some(&observer), + RelayHttpInstrumentation { + observer: Some(&observer), + correlation: None, + }, ) .await; if disconnect_client { @@ -4586,6 +5290,7 @@ mod tests { resolver: &SecretResolver, options: crate::l7::rest::RelayRequestOptions<'_>, ) { + const REQUEST_ID: &str = "1329fced-3bb8-4810-bc25-45a1282997ad"; let (mut client_peer, mut client) = tokio::io::duplex(8192); let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); let ctx = L7EvalContext { @@ -4596,8 +5301,10 @@ mod tests { secret_resolver: Some(Arc::new(resolver.clone())), ..Default::default() }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let outcome = relay_http_request_with_credential_rejection( + let outcome = relay_http_request_with_credential_rejection_observed( &request, &mut client, &mut upstream, @@ -4607,6 +5314,10 @@ mod tests { }, &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .expect("typed credential denial"); @@ -4621,6 +5332,13 @@ mod tests { response.contains("credential_endpoint_mismatch"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert!(body.get("request_id").is_none()); let mut forwarded = Vec::new(); upstream_peer.read_to_end(&mut forwarded).await.unwrap(); assert!( @@ -5210,6 +5928,65 @@ mod tests { assert!(!sent.contains("synthetic-signing-secret")); } + #[tokio::test] + async fn header_endpoint_mismatch_traces_refusal_without_failing_request_root() { + use crate::test_support::request_tracing::{RequestTracing, span_attribute}; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let (_state, resolver) = endpoint_mismatch_resolver(TestHashMap::from([( + "API_TOKEN".to_string(), + "secret".to_string(), + )])); + let request = crate::l7::provider::L7Request { + action: "GET".into(), + target: "/outside".into(), + query_params: TestHashMap::new(), + raw_header: b"GET /outside HTTP/1.1\r\nHost: denied.example.test\r\nAuthorization: Bearer openshell:resolve:env:v1_API_TOKEN\r\nContent-Length: 0\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::ContentLength(0), + }; + Box::pin(assert_credential_relay_rejected( + request, + resolver.as_ref(), + crate::l7::rest::RelayRequestOptions::default(), + )) + .with_subscriber(capture.subscriber()) + .await; + let spans = capture.spans(); + capture.shutdown(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "denied" + ); + assert!(span_attribute(root, "openshell.request.failure_reason").is_none()); + assert!(!matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + let prepare = spans + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .unwrap(); + assert_eq!(prepare.parent_span_id, root.span_context.span_id()); + assert!(matches!( + prepare.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!( + spans + .iter() + .all(|span| span.name != "supervisor.egress.deny") + ); + } + #[tokio::test] async fn body_only_endpoint_mismatch_returns_typed_403() { let (_state, resolver) = endpoint_mismatch_resolver(TestHashMap::from([( @@ -5275,6 +6052,68 @@ mod tests { .await; } + #[tokio::test] + async fn correlated_credential_unavailable_preserves_transport_failure() { + const REQUEST_ID: &str = "250b9326-7a4f-4aa0-a6a9-7e0a1d96beaa"; + let request = crate::l7::provider::L7Request { + action: "GET".to_string(), + target: "/v1/resource".to_string(), + query_params: TestHashMap::new(), + raw_header: b"GET /v1/resource HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "sigv4-policy".into(), + ..Default::default() + }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); + let (mut client_peer, mut client) = tokio::io::duplex(8192); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); + + let error = relay_http_request_with_credential_rejection_observed( + &request, + &mut client, + &mut upstream, + crate::l7::rest::RelayRequestOptions { + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: "execute-api", + signing_region: "us-west-2", + host: "api.example.test", + port: 443, + ..Default::default() + }, + &ctx, + None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, + ) + .await + .expect_err("credential availability failure remains a transport failure"); + assert!( + error + .downcast_ref::() + .is_some() + ); + drop(client); + drop(upstream); + + let mut response = String::new(); + client_peer.read_to_string(&mut response).await.unwrap(); + assert!( + response.is_empty(), + "no new synthetic HTTP response: {response}" + ); + + let mut forwarded = Vec::new(); + upstream_peer.read_to_end(&mut forwarded).await.unwrap(); + assert!(forwarded.is_empty()); + } + fn install_builtin_middleware(engine: &OpaEngine) { engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( openshell_supervisor_middleware_builtins::services() @@ -7819,20 +8658,24 @@ network_policies: enforcement: &str, replacement: &'static [u8], ) -> (String, Option) { + use tracing::instrument::WithSubscriber as _; let (config, tunnel_engine, ctx) = jsonrpc_transforming_relay_parts(enforcement, replacement); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); - let relay = tokio::spawn(async move { - relay_jsonrpc( - &config, - &tunnel_engine, - &mut relay_client, - &mut relay_upstream, - &ctx, - ) - .await - }); + let relay = tokio::spawn( + async move { + relay_jsonrpc( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + .with_current_subscriber(), + ); let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.list"}"#; let request = format!( @@ -7881,32 +8724,80 @@ network_policies: } #[tokio::test] - async fn transformed_jsonrpc_body_is_reevaluated_and_denied() { - // Policy allows reports.list; the middleware replaces the body with a - // method the policy denies. The transformed body must be re-evaluated - // and the request denied before anything reaches the upstream. - let (response, upstream_seen) = run_jsonrpc_transform_case( - "enforce", - br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, - ) - .await; - assert_middleware_failure_response(&response, "jsonrpc_api"); - assert!(upstream_seen.is_none(), "upstream must not see the request"); - } + async fn transformed_jsonrpc_policy_records_keep_their_decisions_and_shared_receipt() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; - #[tokio::test] - async fn transformed_jsonrpc_body_policy_deny_forwards_under_audit() { - // Under enforcement: audit a policy deny of the transformed body is - // logged but forwarded, mirroring audit semantics for original - // bodies. - let (response, upstream_seen) = run_jsonrpc_transform_case( - "audit", - br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, - ) - .await; - assert!(response.contains("204 No Content"), "{response}"); - let upstream_seen = upstream_seen.expect("audited request reaches upstream"); - assert!(upstream_seen.contains("admin.delete"), "{upstream_seen}"); + for enforcement in ["enforce", "audit"] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (response, upstream_seen) = run_jsonrpc_transform_case( + enforcement, + br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, + ) + .with_subscriber(subscriber) + .await; + let request_id = response_request_id(response.as_bytes()); + let events = capture.json_events(); + let (controls, invocations) = assert_http_control_records(&events, 2, 1); + let initial = controls[0]; + let transformed = controls[1]; + let companions = invocations; + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["unmapped"]["transformed"], true); + assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(initial["action"], "Allowed"); + assert!( + initial["message"] + .as_str() + .unwrap() + .starts_with("JSONRPC_L7_REQUEST decision=allow ") + ); + for control in &controls { + assert_eq!(control["metadata"]["correlation_uid"], request_id); + assert_eq!(control["http_request"]["url"]["path"], "/rpc"); + assert_eq!(control["firewall_rule"]["type"], "l7-jsonrpc"); + assert!(control.get("http_response").is_none()); + } + assert!( + transformed["message"] + .as_str() + .unwrap() + .contains("middleware transformation denied") + ); + let sequences: Vec<_> = events + .iter() + .filter(|event| event["class_uid"] == 4002) + .filter_map(|event| event["unmapped"]["request_sequence"].as_u64()) + .collect(); + assert_eq!(sequences.len(), 3, "{events:?}"); + assert!(sequences.windows(2).all(|pair| pair[0] < pair[1])); + if enforcement == "enforce" { + assert_middleware_failure_response(&response, "jsonrpc_api"); + assert!(upstream_seen.is_none(), "upstream must not see the request"); + assert_eq!(transformed["action"], "Denied"); + assert!( + transformed["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST_TRANSFORMED deny ") + ); + } else { + assert!(response.contains("204 No Content"), "{response}"); + assert!( + upstream_seen + .expect("audited request reaches upstream") + .contains("admin.delete") + ); + assert_eq!(transformed["action"], "Allowed"); + assert!( + transformed["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST_TRANSFORMED audit ") + ); + } + } } #[tokio::test] @@ -8775,7 +9666,7 @@ network_policies: denied_http.format_shorthand(), "HTTP:POST [MED] DENIED POST https://api.example.test/v1/messages \ [policy:rest_api engine:middleware] \ - [failed:false transformed:false \ + [failed:false http_event_kind:middleware_invocation middleware_phase:request transformed:false \ reason:middleware_denied:content-guard:content_match]" ); @@ -8794,6 +9685,150 @@ network_policies: assert!(!serialized.contains(RAW_SECRET)); } + #[tokio::test] + async fn passthrough_receipts_require_selected_middleware_and_preserve_control_records() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + for selected in [false, true] { + let middleware = if selected { + "network_middlewares:\n scheme:\n middleware: test/scheme\n on_error: fail_closed\n endpoints:\n include: [api.example.test]\n" + } else { + "" + }; + let engine = OpaEngine::from_strings( + TEST_POLICY, + &format!("{middleware}network_policies: {{}}\n"), + ) + .unwrap(); + engine.set_middleware_runner_for_tests( + openshell_supervisor_middleware::ChainRunner::new(Arc::new(SchemeRecorder( + Arc::new(std::sync::Mutex::new(Vec::new())), + ))), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let state = ProviderCredentialState::from_bound_environment( + 1, + TestHashMap::from([("API_TOKEN".into(), "synthetic-live-credential".into())]), + TestHashMap::new(), + TestHashMap::new(), + TestHashMap::from([( + "API_TOKEN".into(), + StaticCredentialBinding { + endpoints: vec![StaticCredentialEndpointBinding { + host: "api.example.test".into(), + port: 8080, + path: "/v1/**".into(), + }], + credential_identity: "provider-a:API_TOKEN".into(), + workload_credential_handle: String::new(), + }, + )]), + Vec::new(), + ) + .unwrap(); + let placeholder = state.snapshot().child_env["API_TOKEN"].clone(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 8080, + request_default_port: Some(8080), + policy_name: "passthrough_api".into(), + binary_path: "/usr/bin/curl".into(), + provider_credentials: Some(state), + ..Default::default() + }; + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = async move { + relay_passthrough_with_credentials( + &mut relay_client, + &mut relay_upstream, + &ctx, + &generation_guard, + Some(&engine), + ) + .await + } + .with_subscriber(subscriber); + let request = format!( + "GET /v1/messages HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer {placeholder}\r\nX-OpenShell-Request-ID: application-in\r\nConnection: close\r\n\r\n" + ); + let exchange = async { + tokio::join!( + relay, + async { + app.write_all(request.as_bytes()).await.unwrap(); + let mut response = String::new(); + app.read_to_string(&mut response).await.unwrap(); + response + }, + async { + let forwarded = read_http_headers(&mut upstream).await; + upstream.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nX-OpenShell-Request-ID: application-out\r\nConnection: close\r\n\r\n").await.unwrap(); + forwarded + }, + ) + }; + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("credential-only CONNECT exchange"); + relay.unwrap(); + let forwarded = String::from_utf8(forwarded).unwrap(); + assert!( + forwarded.contains("Authorization: Bearer synthetic-live-credential"), + "{forwarded}" + ); + let records = capture.json_events(); + let (controls, companions) = + assert_http_control_records(&records, 1, usize::from(selected)); + let control = controls[0]; + if selected { + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["unmapped"]["failed"], false); + assert_eq!( + companions[0]["metadata"]["correlation_uid"], + control["metadata"]["correlation_uid"] + ); + } + assert_eq!(control["action"], "Allowed"); + assert_eq!(control["http_request"]["url"]["path"], "/v1/messages"); + assert!( + control["message"] + .as_str() + .unwrap() + .starts_with("HTTP_REQUEST ") + ); + assert!(control.get("http_response").is_none()); + assert!( + !serde_json::to_string(&records) + .unwrap() + .contains("synthetic-live-credential") + ); + if selected { + assert!(!forwarded.contains("application-in")); + assert!(!response.contains("application-out")); + let receipt = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(control["metadata"]["correlation_uid"], receipt); + } else { + assert!( + forwarded.contains("X-OpenShell-Request-ID: application-in"), + "{forwarded}" + ); + assert!( + response.contains("X-OpenShell-Request-ID: application-out"), + "{response}" + ); + assert!(control["metadata"].get("correlation_uid").is_none()); + } + } + } + #[tokio::test] async fn passthrough_relay_runs_middleware_redaction() { // A no-protocol endpoint takes the credential-injection passthrough path; @@ -9079,6 +10114,117 @@ network_policies: .unwrap(); } + async fn assert_websocket_endpoint_rejects_plain_http(route_selected: bool) { + let data = r" +network_policies: + ws_api: + name: ws_api + endpoints: + - host: gateway.example.test + port: 443 + path: /ws + protocol: websocket + enforcement: enforce + rules: + - allow: + method: GET + path: /ws + binaries: + - { path: /usr/bin/node } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "gateway.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoints, generation) = engine + .query_endpoint_configs_with_generation(&input) + .unwrap(); + let configs: Vec<_> = endpoints + .iter() + .map(|endpoint| crate::l7::parse_l7_config(endpoint).unwrap()) + .collect(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "gateway.example.test".into(), + port: 443, + request_default_port: Some(443), + policy_name: "ws_api".into(), + binary_path: "/usr/bin/node".into(), + ..Default::default() + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + if route_selected { + relay_with_route_selection( + &configs, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } else { + relay_with_inspection( + &configs[0], + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + }); + + app.write_all( + b"GET /ws HTTP/1.1\r\nHost: gateway.example.test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden"), "{response}"); + let request_id = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); + + let mut upstream_bytes = [0u8; 16]; + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_bytes) + ) + .await, + Err(_) | Ok(Ok(0)) + )); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn websocket_endpoint_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(false).await; + } + + #[tokio::test] + async fn route_selected_websocket_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(true).await; + } + #[tokio::test] async fn passthrough_relay_injects_token_grant_authorization_header() { let (generation_guard, ctx, fixture) = @@ -10142,12 +11288,16 @@ network_policies: .await .unwrap(); - let mut response = [0u8; 1024]; - let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) - .await - .expect("denial should reach client") - .unwrap(); - assert!(String::from_utf8_lossy(&response[..n]).contains("403 Forbidden")); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden")); + let request_id = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); let mut upstream_bytes = [0u8; 16]; assert!(matches!( @@ -11318,7 +12468,7 @@ network_policies: let _guard = capture.install(); let (_app, mut client) = tokio::io::duplex(8192); assert!( - deny_unsupported_upgrade_if_requested(&request, config, &ctx, None, &mut client) + deny_unsupported_upgrade_if_requested(&request, config, &ctx, None, &mut client, None) .await .unwrap() ); @@ -12906,6 +14056,10 @@ network_policies: #[tokio::test] async fn live_request_body_is_completed_only_after_the_stage_finishes() { + use crate::test_support::request_tracing::RequestTracing; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + let data = r#" network_middlewares: echo: @@ -12937,12 +14091,14 @@ network_policies: ("emptied, finished", true, false, Some("")), ("emptied, rejected", true, true, None), ] { + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let preflights = Arc::new(std::sync::Mutex::new(Vec::new())); let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("test policy"); engine.set_middleware_runner_for_tests( openshell_supervisor_middleware::ChainRunner::new(Arc::new(HookEchoRequestStage { drop_body, reject, - preflights: Arc::default(), + preflights: Arc::clone(&preflights), })), ); let input = NetworkInput { @@ -12961,28 +14117,42 @@ network_policies: let tunnel_engine = engine .clone_engine_for_tunnel(generation) .expect("tunnel engine"); + let (child_env, resolver) = SecretResolver::from_provider_env(TestHashMap::from([( + "API_TOKEN".to_string(), + "synthetic-live-token".to_string(), + )])); + let credentialed = !reject && !drop_body; let ctx = L7EvalContext { host: "api.example.test".into(), port: 8080, request_default_port: Some(8080), policy_name: "rest_api".into(), binary_path: "/usr/bin/curl".into(), + secret_resolver: credentialed.then(|| Arc::new(resolver.unwrap())), ..Default::default() }; let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); - let relay = tokio::spawn(async move { - relay_rest( - &config, - &tunnel_engine, - &mut relay_client, - &mut relay_upstream, - &ctx, - ) - .await - }); + let relay = tokio::spawn( + async move { + relay_rest( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + .with_subscriber(capture.subscriber()), + ); + let authorization = if credentialed { + format!("Authorization: Bearer {}\r\n", child_env["API_TOKEN"]) + } else { + String::new() + }; app.write_all( - b"POST /v1/items HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 5\r\n\r\nhello", + format!("POST /v1/items HTTP/1.1\r\nHost: api.example.test\r\n{authorization}Content-Length: 5\r\n\r\nhello").as_bytes(), ) .await .expect("send request"); @@ -13024,6 +14194,53 @@ network_policies: let forwarded = String::from_utf8(forwarded).unwrap(); let response = String::from_utf8(response).unwrap(); + let request_id = response_request_id(response.as_bytes()); + assert_eq!( + preflights.lock().unwrap()[0] + .context + .as_ref() + .unwrap() + .request_id, + request_id + ); + let events = capture.events.json_events(); + let (controls, companions) = assert_http_control_records(&events, 1, 1); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); + assert_eq!( + companions[0]["action"], + if reject { "Denied" } else { "Allowed" } + ); + assert_eq!(controls[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(controls[0]["action"], "Allowed"); + assert!(controls[0].get("http_response").is_none()); + let spans = capture.spans(); + let stage = |stage_name: &str| { + spans + .iter() + .filter(|span| span.name == stage_name) + .collect::>() + }; + assert_eq!( + stage("openshell.middleware.request").len(), + 1, + "{name}: {spans:?}" + ); + assert_eq!( + stage("openshell.credential.prepare").len(), + usize::from(credentialed), + "{name}: {spans:?}" + ); + if !reject && !drop_body { + let sent = stage("openshell.upstream.send"); + let received = stage("openshell.http.response"); + assert_eq!(sent.len(), 1); + assert_eq!(received.len(), 1); + assert!(sent[0].end_time <= received[0].end_time); + } + assert!(!format!("{spans:?}").contains("synthetic-live-token")); + assert!(!format!("{events:?}").contains("synthetic-live-token")); + capture.shutdown(); match forwarded_body { Some(body) => { let (head, forwarded_body) = forwarded @@ -14002,6 +15219,7 @@ network_policies: &ctx, "/mcp", None, + None, ) .await .unwrap(); @@ -14034,7 +15252,8 @@ network_policies: &mut response, &ctx, "/mcp", - None + None, + None, ) .await .unwrap() @@ -14485,6 +15704,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await; if disconnect_client { @@ -14597,6 +15817,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await .expect("initialize version exemption") @@ -14621,6 +15842,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await; if disconnect_client { @@ -14709,6 +15931,7 @@ network_policies: #[tokio::test] async fn final_mcp_version_check_reclassifies_a_rewritten_initialize_body() { + const REQUEST_ID: &str = "36a15952-692c-44d8-a373-eeca9735fb6b"; let (config, _, ctx) = mcp_test_relay_context(); let final_body = br#"{"jsonrpc":"2.0","id":2,"method":"tools/list"}"#; let mut raw_header = format!( @@ -14725,6 +15948,8 @@ network_policies: body_length: crate::l7::provider::BodyLength::ContentLength(final_body.len() as u64), }; let (mut client, mut relay_client) = tokio::io::duplex(2048); + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); let allowed = enforce_final_mcp_protocol_version( &config, @@ -14733,6 +15958,7 @@ network_policies: &ctx, "/mcp", None, + Some(&correlation), ) .await .expect("final request inspection"); @@ -14740,6 +15966,7 @@ network_policies: !allowed, "rewritten non-initialize request must require a version" ); + drop(correlation); drop(relay_client); let mut response = Vec::new(); @@ -14750,6 +15977,13 @@ network_policies: response.contains("mcp_protocol_version_not_allowed"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let response_body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert!(response_body.get("request_id").is_none()); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs new file mode 100644 index 0000000000..4128083134 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -0,0 +1,1279 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Correlation contracts for the production single-config relay dispatch. + +use super::L7EvalContext; +use crate::l7::L7EndpointConfig; +use crate::opa::{NetworkInput, OpaEngine, TunnelPolicyEngine}; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use crate::test_support::response_request_id as request_id; +use opentelemetry_sdk::trace::{Sampler, SpanData}; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::layer::SubscriberExt as _; + +const POLICY: &str = include_str!("../../../data/sandbox-policy.rego"); + +pub struct TracedOperation { + pub(crate) output: T, + pub(crate) events: Vec, + pub(crate) spans: Vec, + created: Vec<&'static str>, +} + +impl TracedOperation { + pub(crate) fn assert_no_deny_span(&self, sampled: bool) { + assert!(!self.created.contains(&"supervisor.egress.deny")); + assert_eq!( + self.created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1 + ); + let roots: Vec<_> = self + .spans + .iter() + .filter(|span| span.name == "openshell.http.request") + .collect(); + assert_eq!(roots.len(), usize::from(sampled)); + if let Some(root) = roots.first() { + let root_id = root.span_context.span_id(); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + let by_id = self + .spans + .iter() + .map(|span| (span.span_context.span_id(), span)) + .collect::>(); + for span in &self.spans { + assert_eq!(span.span_context.trace_id(), root.span_context.trace_id()); + if span.span_context.span_id() == root_id { + continue; + } + let mut parent = span.parent_span_id; + for _ in 0..self.spans.len() { + if parent == root_id { + break; + } + parent = by_id + .get(&parent) + .expect("span parent is captured") + .parent_span_id; + } + assert_eq!( + parent, root_id, + "span parent graph reaches the request root" + ); + if matches!( + span.name.as_ref(), + "openshell.policy.evaluate" + | "openshell.middleware.request" + | "openshell.middleware.websocket.preflight" + | "openshell.credential.prepare" + | "openshell.upstream.resolve" + | "openshell.upstream.connect" + | "openshell.upstream.send" + | "openshell.http.response" + | "openshell.middleware.response" + ) { + assert_eq!(span.parent_span_id, root_id, "request stage parent"); + } + } + for event in self + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + { + assert_eq!( + event["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + } else { + assert!(self.spans.is_empty()); + assert!(self.events.iter().all(|event| event.get("trace").is_none())); + } + } +} + +pub async fn capture_request_spans( + sampled: bool, + operation: impl Future, +) -> TracedOperation { + capture_request( + Some(if sampled { + Sampler::AlwaysOn + } else { + Sampler::AlwaysOff + }), + operation, + ) + .await +} + +async fn capture_request( + sampler: Option, + operation: impl Future, +) -> TracedOperation { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let capture = RequestTracing::new(sampler); + let output = Box::pin(operation.with_subscriber(capture.subscriber())).await; + let spans = capture.spans(); + capture.shutdown(); + TracedOperation { + output, + events: capture.events.json_events(), + spans, + created: capture.created_span_names(), + } +} + +#[tokio::test] +async fn request_trace_capture_accepts_rpc_grandchildren() { + let evidence = capture_request_spans(true, async { + let request = tracing::info_span!(parent: None, "openshell.http.request"); + request.in_scope(|| { + tracing::info_span!("openshell.middleware.request").in_scope(|| { + tracing::info_span!("middleware.rpc").in_scope(|| {}); + }); + }); + }) + .await; + assert_eq!(evidence.spans.len(), 3); + evidence.assert_no_deny_span(true); +} + +struct Fixture { + config: L7EndpointConfig, + tunnel_engine: TunnelPolicyEngine, + middleware_engine: OpaEngine, + ctx: L7EvalContext, +} + +impl Fixture { + fn with_middleware_runner( + mut self, + runner: openshell_supervisor_middleware::ChainRunner, + ) -> Self { + self.middleware_engine + .set_middleware_runner_for_tests(runner); + // Tunnel engines snapshot the runner alongside the policy generation. + self.tunnel_engine = self + .middleware_engine + .clone_engine_for_tunnel(self.middleware_engine.current_generation()) + .expect("pin injected middleware runner"); + self + } +} + +struct Exchange { + response: Vec, + forwarded: Vec, +} + +fn fixture(data: &str, host: &str, policy_name: &str) -> Fixture { + let middleware_engine = OpaEngine::from_strings(POLICY, data).expect("valid test policy"); + let input = NetworkInput { + host: host.into(), + port: 8000, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = middleware_engine + .query_endpoint_config_with_generation(&input) + .expect("query endpoint config"); + let config = crate::l7::parse_l7_config( + &endpoint_config.expect("single configured endpoint should match"), + ) + .expect("parse endpoint config"); + let tunnel_engine = middleware_engine + .clone_engine_for_tunnel(generation) + .expect("pin test policy generation"); + let ctx = L7EvalContext { + host: host.into(), + port: 8000, + request_default_port: Some(8000), + policy_name: policy_name.into(), + binary_path: "/usr/bin/python3".into(), + ..Default::default() + }; + Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } +} + +async fn read_http_message(reader: &mut R) -> Vec { + let mut message = Vec::new(); + let mut byte = [0_u8; 1]; + while !message.ends_with(b"\r\n\r\n") { + reader + .read_exact(&mut byte) + .await + .expect("complete HTTP headers"); + message.push(byte[0]); + } + let headers = std::str::from_utf8(&message).expect("ASCII test headers"); + let body_len = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length").then(|| { + value + .trim() + .parse::() + .expect("numeric content length") + }) + }) + .unwrap_or_default(); + let header_len = message.len(); + message.resize(header_len + body_len, 0); + reader + .read_exact(&mut message[header_len..]) + .await + .expect("complete HTTP body"); + message +} + +async fn run_allowed( + fixture: Fixture, + request: Vec, + upstream_response: &'static [u8], +) -> TracedOperation { + capture_request( + None, + Box::pin(run_exchange(fixture, request, Some(upstream_response))), + ) + .await +} + +async fn run_denied(fixture: Fixture, request: Vec) -> TracedOperation { + capture_request(None, Box::pin(run_exchange(fixture, request, None))).await +} + +async fn run_exchange( + fixture: Fixture, + request: Vec, + upstream_response: Option<&'static [u8]>, +) -> Exchange { + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = fixture; + + let exchange = async { + tracing::callsite::rebuild_interest_cache(); + tokio::join!( + async { + let result = crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ) + .await; + drop(relay_upstream); + result + }, + async { + app.write_all(&request).await.expect("send request"); + app.flush().await.expect("flush request"); + read_http_message(&mut app).await + }, + async { + if let Some(response) = upstream_response { + let forwarded = read_http_message(&mut upstream).await; + upstream + .write_all(response) + .await + .expect("send upstream response"); + upstream.flush().await.expect("flush upstream response"); + forwarded + } else { + let mut forwarded = Vec::new(); + upstream + .read_to_end(&mut forwarded) + .await + .expect("relay closes upstream"); + forwarded + } + }, + ) + }; + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("single-config relay exchange completes"); + relay.expect("relay succeeds"); + Exchange { + response, + forwarded, + } +} + +fn assert_control_records<'a>( + exchange: &'a TracedOperation, + expected: &[(&str, &str)], + finding_uids: &[&str], + path: &str, +) -> Vec<&'a serde_json::Value> { + let receipt = request_id(&exchange.output.response); + assert!( + uuid::Uuid::parse_str(receipt).is_ok(), + "canonical receipt: {receipt}" + ); + assert_eq!( + exchange.events.len(), + expected.len() + finding_uids.len(), + "{:?}", + exchange.events + ); + for (sequence, event) in exchange.events.iter().enumerate() { + assert_eq!(event["metadata"]["correlation_uid"], receipt); + assert_eq!(event["unmapped"]["request_sequence"], sequence + 1); + } + for (sequence, (event, (engine_type, action))) in + exchange.events.iter().zip(expected).enumerate() + { + assert_eq!(event["class_uid"], 4002); + assert_eq!(event["unmapped"]["request_sequence"], sequence + 1); + assert_eq!(event["firewall_rule"]["type"], *engine_type); + assert_eq!(event["action"], *action); + assert_eq!(event["http_request"]["url"]["path"], path); + assert!( + event.get("http_response").is_none(), + "delivery status must not be overlaid on a control decision: {event}" + ); + } + for (event, uid) in exchange.events[expected.len()..].iter().zip(finding_uids) { + assert_eq!(event["class_uid"], 2004); + assert_eq!(event["finding_info"]["uid"], *uid); + assert_eq!(event["action"], "Denied"); + assert!(event.get("trace").is_none()); + } + exchange.events[..expected.len()].iter().collect() +} + +fn assert_policy_record<'a>( + exchange: &'a TracedOperation, + engine_type: &str, + action: &str, + path: &str, +) -> &'a serde_json::Value { + let event = assert_control_records(exchange, &[(engine_type, action)], &[], path)[0]; + assert_eq!(event["unmapped"]["http_event_kind"], "decision"); + assert!(event.get("status").is_none()); + assert!( + event.get("trace").is_none(), + "receipts work without tracing" + ); + event +} + +fn mcp_fixture() -> Fixture { + fixture( + r#" +network_policies: + mcp_api: + name: mcp_api + endpoints: + - host: mcp.example.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + versions: ["2025-11-25"] + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/python3 } +"#, + "mcp.example.test", + "mcp_api", + ) +} + +fn request(host: &str, path: &str, extra_headers: &str, body: &str) -> Vec { + format!( + "POST {path} HTTP/1.1\r\nHost: {host}:8000\r\nContent-Type: application/json\r\n{extra_headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ) + .into_bytes() +} + +#[derive(Default)] +struct BlockingRequestMiddleware { + http_hooks: bool, +} + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for BlockingRequestMiddleware { + async fn describe(&self) -> openshell_core::proto::MiddlewareManifest { + use openshell_core::proto::{ + MiddlewareBinding, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, + }; + let name = if self.http_hooks { + "test/block-request-hooks" + } else { + "test/block-request" + }; + openshell_core::proto::MiddlewareManifest { + name: name.into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + extension: Some(if self.http_hooks { + openshell_core::extension_protocol::http_hooks_middleware_metadata(name, "test") + } else { + openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + name, + "test", + [], + ) + }), + ..Default::default() + } + } + + async fn validate_config(&self, _: &str, _: &prost_types::Struct) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(openshell_core::proto::HttpRequestResult { + decision: openshell_core::proto::Decision::Deny as i32, + reason_code: "test_block".into(), + ..Default::default() + }) + } +} + +fn rest_fixture(method: &str, block_middleware: bool) -> Fixture { + let middleware = if block_middleware { + "network_middlewares:\n request:\n middleware: test/block-request\n on_error: fail_closed\n endpoints:\n include: [rest.example.test]\n" + } else { + "" + }; + let fixture = rest_fixture_with_middlewares(method, middleware); + if block_middleware { + fixture.with_middleware_runner(openshell_supervisor_middleware::ChainRunner::new(Arc::new( + BlockingRequestMiddleware::default(), + ))) + } else { + fixture + } +} + +fn rest_fixture_with_middlewares(method: &str, middleware: &str) -> Fixture { + fixture( + &format!( + r"{middleware} +network_policies: + rest_api: + name: rest_api + endpoints: + - host: rest.example.test + port: 8000 + path: /item + protocol: rest + enforcement: enforce + rules: + - allow: {{ method: {method}, path: /item }} + binaries: + - {{ path: /usr/bin/python3 }} +" + ), + "rest.example.test", + "rest_api", + ) +} + +#[tokio::test] +async fn inspected_allow_creates_only_the_stages_that_run() { + for sampled in [true, false] { + let evidence = capture_request_spans( + sampled, + Box::pin(run_exchange( + rest_fixture("POST", false), + request("rest.example.test", "/item", "", "{}"), + Some(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), + )), + ) + .await; + evidence.assert_no_deny_span(sampled); + assert!( + evidence + .output + .forwarded + .starts_with(b"POST /item HTTP/1.1\r\n") + ); + if sampled { + let names: Vec<_> = evidence + .spans + .iter() + .map(|span| span.name.as_ref()) + .collect(); + for (name, expected) in [ + ("openshell.policy.evaluate", true), + ("openshell.upstream.send", true), + ("openshell.http.response", true), + ("openshell.middleware.request", false), + ("openshell.middleware.response", false), + ("openshell.credential.prepare", false), + ] { + assert_eq!(names.contains(&name), expected, "{name}: {names:?}"); + } + } + let records = assert_control_records(&evidence, &[("l7", "Allowed")], &[], "/item"); + assert!(records[0].get("status").is_none()); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") + ); + } +} + +#[tokio::test] +async fn request_middleware_block_never_creates_a_denial_only_span() { + for (sampled, service_registered) in [(true, false), (false, false), (true, true)] { + let fixture = rest_fixture("POST", true); + let fixture = if service_registered { + fixture + } else { + fixture.with_middleware_runner(openshell_supervisor_middleware::ChainRunner::default()) + }; + let evidence = capture_request_spans( + sampled, + Box::pin(run_exchange( + fixture, + request("rest.example.test", "/item", "", "{}"), + None, + )), + ) + .await; + evidence.assert_no_deny_span(sampled); + assert!(evidence.output.forwarded.is_empty()); + assert!( + evidence + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n") + ); + if sampled { + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.request") + .collect(); + assert_eq!(middleware.len(), 1); + if service_registered { + assert_eq!(middleware[0].status, opentelemetry::trace::Status::Ok); + } else { + assert!(matches!( + middleware[0].status, + opentelemetry::trace::Status::Error { .. } + )); + } + let reason = (!service_registered).then_some("request_middleware_failed"); + assert_eq!( + span_attribute(middleware[0], "openshell.request.failure_reason") + .map(|value| value.as_str()), + reason.map(std::borrow::Cow::Borrowed) + ); + assert_request_outcome( + &evidence, + request_id(&evidence.output.response), + if service_registered { + "denied" + } else { + "failed" + }, + reason, + ); + assert!( + evidence + .spans + .iter() + .all(|span| span.name != "openshell.upstream.send") + ); + } + let records = assert_control_records( + &evidence, + &[("l7", "Allowed"), ("middleware", "Denied")], + if service_registered { + &[] + } else { + &["openshell.middleware.failure"] + }, + "/item", + ); + assert_eq!(records[0]["unmapped"]["http_event_kind"], "decision"); + assert!(records[0].get("status").is_none()); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") + ); + assert_eq!( + records[1]["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(records[1]["unmapped"]["middleware_phase"], "request"); + assert_eq!(records[1]["unmapped"]["failed"], !service_registered); + assert_eq!(records[1]["status"], "Failure"); + assert_eq!( + records[1]["status_detail"], + if service_registered { + "middleware_denied:request:test_block" + } else { + "middleware_failed: binding_not_described" + } + ); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("MIDDLEWARE request test/block-request decision=Deny") + ); + } +} + +#[tokio::test] +async fn mixed_request_middleware_hooks_fail_the_stage_and_request() { + let fixture = rest_fixture_with_middlewares( + "POST", + r"network_middlewares: + legacy: + middleware: test/block-request + order: 0 + on_error: fail_closed + endpoints: + include: [rest.example.test] + current: + middleware: test/block-request-hooks + order: 1 + on_error: fail_closed + endpoints: + include: [rest.example.test] +", + ); + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + vec![ + Arc::new(BlockingRequestMiddleware::default()), + Arc::new(BlockingRequestMiddleware { http_hooks: true }), + ], + vec![], + ) + .await + .expect("register both hook versions"); + let fixture = fixture.with_middleware_runner( + openshell_supervisor_middleware::ChainRunner::from_registry(registry), + ); + let evidence = capture_request_spans( + true, + Box::pin(run_exchange( + fixture, + request("rest.example.test", "/item", "", "{}"), + None, + )), + ) + .await; + + assert!(evidence.output.forwarded.is_empty()); + assert!( + evidence + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n"), + "response: {}", + String::from_utf8_lossy(&evidence.output.response) + ); + assert_request_outcome( + &evidence, + request_id(&evidence.output.response), + "failed", + Some("request_middleware_failed"), + ); + let records = assert_control_records( + &evidence, + &[("l7", "Allowed"), ("middleware", "Denied")], + &["openshell.middleware.failure"], + "/item", + ); + assert_eq!( + records[1]["status_detail"], + "middleware_failed: middleware_hook_versions_mixed" + ); + let middleware = evidence + .spans + .iter() + .find(|span| span.name == "openshell.middleware.request") + .expect("request middleware stage"); + assert!(matches!( + middleware.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(middleware, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "request_middleware_failed" + ); +} + +#[tokio::test] +async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let exchange = async { + tokio::join!( + crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ), + async { + let mut responses = Vec::new(); + for (sampled, connection) in [("01", "keep-alive"), ("00", "close")] { + let request = format!( + "GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-{sampled}\r\nConnection: {connection}\r\n\r\n", + ); + app.write_all(request.as_bytes()).await.expect("send request"); + app.flush().await.expect("flush request"); + responses.push(read_http_message(&mut app).await); + } + responses + }, + async { + for connection in ["keep-alive", "close"] { + let request = read_http_message(&mut upstream).await; + assert!(request.starts_with(b"GET /item HTTP/1.1\r\n")); + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: {connection}\r\n\r\n"); + upstream.write_all(response.as_bytes()).await.expect("upstream response"); + upstream.flush().await.expect("flush response"); + } + }, + ) + }.with_subscriber(subscriber); + let (relay, responses, ()) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("two inspected requests complete"); + relay.expect("inspected relay"); + let receipts: Vec<_> = responses + .iter() + .map(|response| request_id(response)) + .collect(); + assert_eq!(receipts.len(), 2); + assert_ne!(receipts[0], receipts[1]); + let records: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!( + records.len(), + 2, + "one policy decision per request: {records:?}" + ); + for (record, receipt) in records.iter().zip(receipts) { + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(record["metadata"]["correlation_uid"], receipt); + assert_eq!(record["action"], "Allowed"); + assert!(record.get("http_response").is_none()); + assert!(record.get("status").is_none()); + assert_eq!(record["firewall_rule"]["type"], "l7"); + assert_eq!(record["unmapped"]["http_event_kind"], "decision"); + assert_eq!(record["unmapped"]["request_sequence"], 1); + assert_eq!(record["http_request"]["url"]["path"], "/item"); + assert!(record.get("trace").is_none()); + assert!( + !record + .to_string() + .contains("4bf92f3577b34da6a3ce929d0e0e4736") + ); + } +} + +#[tokio::test] +async fn keep_alive_parse_rejection_does_not_reuse_the_completed_request_scope() { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let evidence = capture_request_spans( + true, + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + let forwarded = tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + request = read_http_message(&mut upstream) => request, + }; + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + let response = tokio::select! { + result = relay.as_mut() => panic!("persistent relay stopped: {result:?}"), + response = read_http_message(&mut app) => response, + }; + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + let receipt = request_id(&response).to_string(); + assert!(futures::poll!(relay.as_mut()).is_pending()); + app.write_all(b"GET /item?q=%ff HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + relay.await.expect("parse rejection closes the relay"); + receipt + }), + ) + .await; + let receipt = evidence + .output + .as_ref() + .expect("keep-alive exchange completes"); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + evidence.assert_no_deny_span(true); + let (rejected, completed): (Vec<_>, Vec<_>) = evidence.events.iter().partition(|event| { + event["message"] + == "HTTP request rejected before policy evaluation for rest.example.test:8000" + }); + assert!(!completed.is_empty()); + for event in completed { + assert_eq!(event["metadata"]["correlation_uid"], receipt.as_str()); + } + assert_eq!(rejected.len(), 1, "actual parse rejection: {rejected:?}"); + let rejection = rejected[0]; + assert_eq!(rejection["class_uid"], 4001); + assert_eq!(rejection["action"], "Denied"); + assert_eq!( + rejection["status_detail"], + "Query component is not valid UTF-8" + ); + assert!(rejection["metadata"]["correlation_uid"].is_null()); + assert!(rejection["unmapped"]["request_sequence"].is_null()); + assert!(rejection["unmapped"]["http_event_kind"].is_null()); + assert!(rejection["trace"]["uid"].is_null()); +} + +#[derive(Clone, Copy, Debug)] +enum ReloadAt { + BeforeRequest, + DuringResponse, + BetweenRequests, +} + +async fn reload_inspected_relay(at: ReloadAt) -> Option { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + let mut receipt = None; + if !matches!(at, ReloadAt::BeforeRequest) { + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + forwarded = read_http_message(&mut upstream) => { + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + } + } + if matches!(at, ReloadAt::BetweenRequests) { + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + let response = tokio::select! { + result = relay.as_mut() => panic!("persistent relay stopped: {result:?}"), + response = read_http_message(&mut app) => response, + }; + receipt = Some(request_id(&response).to_string()); + } + } + // Drive the real relay into the pending response read or next request + // read before reloading, without a scheduling delay or a test-only hook. + assert!(futures::poll!(relay.as_mut()).is_pending(), "{at:?}"); + middleware_engine + .reload(POLICY, "network_policies: {}\n") + .expect("reload closes the generation-pinned relay"); + relay.await.expect("stale relay closes without a new error"); + receipt +} + +fn assert_request_outcome( + evidence: &TracedOperation, + receipt: &str, + outcome: &str, + reason: Option<&str>, +) { + evidence.assert_no_deny_span(true); + let root = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.id") + .unwrap() + .as_str(), + receipt + ); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + outcome, + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason").map(|value| value.as_str()), + reason.map(std::borrow::Cow::Borrowed), + ); + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + outcome == "failed", + ); +} + +#[tokio::test] +async fn production_policy_reload_correlates_only_the_active_request() { + for at in [ + ReloadAt::BeforeRequest, + ReloadAt::DuringResponse, + ReloadAt::BetweenRequests, + ] { + let evidence = capture_request_spans( + true, + tokio::time::timeout( + std::time::Duration::from_secs(5), + Box::pin(reload_inspected_relay(at)), + ), + ) + .await; + let completed_receipt = evidence.output.as_ref().expect("policy reload completes"); + let started = !matches!(at, ReloadAt::BeforeRequest); + let events: Vec<_> = evidence + .events + .iter() + .filter(|event| matches!(event["class_uid"].as_u64(), Some(4001 | 4002))) + .collect(); + assert_eq!(events.len(), 1 + usize::from(started), "{at:?}"); + let close = events.last().unwrap(); + assert_eq!(close["class_uid"], 4001); + assert_eq!(close["action"], "Denied"); + assert_eq!(close["status"], "Failure"); + assert!( + close["message"] + .as_str() + .unwrap() + .starts_with("L7 tunnel closed before inspection because policy changed:") + ); + if matches!(at, ReloadAt::DuringResponse) { + let policy = events[0]; + let receipt = policy["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(policy["action"], "Allowed"); + assert_eq!(close["metadata"]["correlation_uid"], receipt); + assert_eq!(close["unmapped"]["request_sequence"], 2); + assert_eq!(close["unmapped"]["http_event_kind"], "connection"); + assert_request_outcome(&evidence, receipt, "failed", Some("policy_reloaded")); + } else { + assert!( + close["metadata"].get("correlation_uid").is_none(), + "{at:?}: {close}" + ); + assert!( + close["unmapped"].get("request_sequence").is_none(), + "{at:?}: {close}" + ); + if let Some(receipt) = completed_receipt { + assert_eq!(events[0]["metadata"]["correlation_uid"], *receipt); + assert_request_outcome(&evidence, receipt, "allowed", None); + } else { + assert!(evidence.spans.is_empty()); + assert!(!evidence.created.contains(&"openshell.http.request")); + } + } + } +} + +#[tokio::test] +async fn oversized_response_header_keeps_request_attribution_after_dispatch_returns_error() { + let operation = async { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + forwarded = read_http_message(&mut upstream) => { + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + } + } + let oversized = format!( + "HTTP/1.1 200 OK\r\nX-Fill: {}\r\n\r\n", + "x".repeat(16 * 1024) + ); + upstream.write_all(oversized.as_bytes()).await.unwrap(); + let error = relay + .await + .expect_err("oversized upstream response headers fail closed"); + assert!( + error + .to_string() + .contains("HTTP response headers exceed limit") + ); + + // This is the existing outer plaintext relay-error record, emitted + // only after the production dispatch has dropped its request guard. + let message = format!("L7 relay error: {error}"); + let terminal = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Fail) + .severity(openshell_ocsf::SeverityId::Low) + .status(openshell_ocsf::StatusId::Failure) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain(&ctx.host, ctx.port)) + .message(&message) + .build(); + let event_uid = terminal.to_json().unwrap()["metadata"]["uid"].clone(); + crate::l7::correlation::emit_error_event(terminal, &error); + (message, event_uid) + }; + let evidence = capture_request_spans( + true, + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(operation)), + ) + .await; + let (message, event_uid) = evidence + .output + .as_ref() + .expect("oversized response header fails without waiting for EOF"); + let events: Vec<_> = evidence + .events + .iter() + .filter(|event| matches!(event["class_uid"].as_u64(), Some(4001 | 4002))) + .collect(); + assert_eq!(events.len(), 2, "{:?}", evidence.events); + let policy = events[0]; + let terminal = events[1]; + let receipt = policy["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(policy["class_uid"], 4002); + assert_eq!(policy["action"], "Allowed"); + assert_eq!(terminal["class_uid"], 4001); + assert_eq!(terminal["metadata"]["correlation_uid"], receipt); + assert_eq!(terminal["metadata"]["uid"], *event_uid); + assert_eq!(terminal["unmapped"]["request_sequence"], 2); + assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); + assert_eq!(terminal["message"], *message); + assert_eq!(terminal["status"], "Failure"); + assert!(terminal.get("action").is_none()); + assert!(terminal.get("disposition").is_none()); + assert_request_outcome(&evidence, receipt, "failed", Some("response_failed")); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert!(matches!( + response.status, + opentelemetry::trace::Status::Error { .. } + )); +} + +#[tokio::test] +async fn production_single_mcp_allow_correlates_record_and_response_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_allowed( + mcp_fixture(), + request("mcp.example.test", "/mcp", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + )) + .await; + + assert!( + exchange + .output + .forwarded + .starts_with(b"POST /mcp HTTP/1.1\r\n") + ); + assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_policy_record(&exchange, "l7-mcp", "Allowed", "/mcp"); +} + +#[tokio::test] +async fn production_single_mcp_invalid_version_correlates_denial_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 1900-01-01\r\n", + body, + ), + )) + .await; + + assert!(exchange.output.forwarded.is_empty()); + assert!( + exchange + .output + .response + .starts_with(b"HTTP/1.1 400 Bad Request\r\n") + ); + assert!( + String::from_utf8_lossy(&exchange.output.response) + .contains("unsupported_mcp_protocol_version") + ); + assert_policy_record(&exchange, "l7-mcp", "Denied", "/mcp"); +} + +#[tokio::test] +async fn production_single_mcp_policy_deny_correlates_record_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 2025-11-25\r\n", + body, + ), + )) + .await; + + assert!(exchange.output.forwarded.is_empty()); + assert!( + exchange + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n") + ); + let event = assert_policy_record(&exchange, "l7-mcp", "Denied", "/mcp"); + assert!( + event["message"] + .as_str() + .is_some_and(|message| message.starts_with("JSONRPC_L7_REQUEST decision=deny")) + ); +} + +#[tokio::test] +async fn production_single_graphql_allow_correlates_record_and_response_receipt() { + let graphql = fixture( + r" +network_policies: + graphql_api: + name: graphql_api + endpoints: + - host: graphql.example.test + port: 8000 + path: /graphql + protocol: graphql + enforcement: enforce + rules: + - allow: + operation_type: query + fields: [viewer] + binaries: + - { path: /usr/bin/python3 } +", + "graphql.example.test", + "graphql_api", + ); + let body = r#"{"query":"query { viewer }"}"#; + let exchange = Box::pin(run_allowed( + graphql, + request("graphql.example.test", "/graphql", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 22\r\nConnection: close\r\n\r\n{\"data\":{\"viewer\":{}}}", + )) + .await; + + assert!( + exchange + .output + .forwarded + .starts_with(b"POST /graphql HTTP/1.1\r\n") + ); + assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_policy_record(&exchange, "l7-graphql", "Allowed", "/graphql"); +} diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 19a9633fb0..d53b1e0abd 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -31,6 +31,7 @@ use tokio::io::{AsyncReadExt, DuplexStream, ReadBuf}; use tokio::sync::Notify; use tokio::task::JoinHandle; use tokio::time::timeout; +use tracing::instrument::WithSubscriber as _; const POLICY: &str = include_str!("../../../data/sandbox-policy.rego"); const HOST: &str = "api.example.test"; @@ -104,7 +105,6 @@ struct Route<'a> { method: &'a str, binary: &'a str, enforcement: &'a str, - deny_path: Option<&'a str>, body_rewrite: bool, protocol: &'a str, } @@ -116,7 +116,6 @@ fn route<'a>(name: &'a str, path: &'a str) -> Route<'a> { method: "GET", binary: "/usr/bin/curl", enforcement: "enforce", - deny_path: None, body_rewrite: false, protocol: "rest", } @@ -186,10 +185,6 @@ impl Fixture { if route.protocol == "mcp" { endpoint["mcp"] = serde_json::json!({"versions": ["2025-11-25"]}); } - // An explicit empty deny_rules list is invalid authored policy. - if let Some(path) = route.deny_path { - endpoint["deny_rules"] = serde_json::json!([{"method": "GET", "path": path}]); - } if route.body_rewrite { endpoint["request_body_credential_rewrite"] = true.into(); } @@ -246,11 +241,7 @@ impl Fixture { } fn grant_key(&self, path: &str, owner: &str) -> String { - let snapshot = self.state.snapshot(); - format!( - "{HOST}\t{PORT}\t{path}\trev:{}\tinstallation:{}\t{owner}:access_token", - snapshot.revision, snapshot.installation_id - ) + self.state.snapshot().scoped_key(&key(path, owner)) } fn reload_policy(&mut self) { @@ -304,17 +295,20 @@ impl Fixture { notified: false, }; let (mut relay_upstream, upstream) = tokio::io::duplex(16 * 1024); - let relay = tokio::spawn(async move { - crate::proxy::relay_inspected_http_stream_for_test( - &mut client, - &mut relay_upstream, - configs, - engine, - &middleware_engine, - &ctx, - ) - .await - }); + let relay = tokio::spawn( + async move { + crate::proxy::relay_inspected_http_stream_for_test( + &mut client, + &mut relay_upstream, + configs, + engine, + &middleware_engine, + &ctx, + ) + .await + } + .with_current_subscriber(), + ); Connection { app, relay, @@ -1003,6 +997,70 @@ async fn failure_then_reconnect(route_selection: bool) { ]); } +#[tokio::test] +async fn token_grant_refusal_never_creates_a_denial_only_span() { + use super::correlation_dispatch_tests::capture_request_spans; + + for sampled in [true, false] { + let evidence = capture_request_spans(sampled, async { + let fixture = Fixture::new(&[route("a", "/a/**")], &[("/a/**", "a", "aud-a")], 1); + fixture.resolver.fail_once.store(true, Ordering::SeqCst); + let mut connection = fixture.connect(); + let response = connection.exchange("GET", "/a/item", true).await; + assert!(connection.observes_eof().await); + let capture = connection.finish().await; + assert_eq!(capture.bytes_seen, 0); + response + }) + .await; + evidence.assert_no_deny_span(sampled); + assert!(evidence.output.starts_with("HTTP/1.1 502")); + if sampled { + assert_eq!( + evidence + .spans + .iter() + .filter(|span| span.name == "openshell.credential.prepare") + .count(), + 1 + ); + assert!( + evidence + .spans + .iter() + .all(|span| span.name != "openshell.upstream.send") + ); + } + let records: Vec<_> = evidence + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(records.len(), 2, "{:?}", evidence.events); + assert_eq!(records[0]["action"], "Allowed"); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") + ); + assert_eq!(records[1]["action"], "Denied"); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("Token grant failed ") + ); + for record in records { + assert!(record.get("http_response").is_none()); + assert_eq!( + record["metadata"]["correlation_uid"], + crate::test_support::response_request_id(evidence.output.as_bytes()) + ); + } + } +} + #[tokio::test] async fn single_endpoint_recovers_after_one_grant_failure() { failure_then_reconnect(false).await; @@ -1418,6 +1476,7 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { &info, "request-final-body", None, + None, ), ) .await @@ -1431,11 +1490,7 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { std::str::from_utf8(&injected.raw_header).expect("injected HTTP request"), "aud-transformed", ); - let snapshot = state.snapshot(); - let expected_key = format!( - "{HOST}\t{PORT}\t/api/**\trev:{}\tinstallation:{}\ttransformed:access_token", - snapshot.revision, snapshot.installation_id - ); + let expected_key = state.snapshot().scoped_key(&key("/api/**", "transformed")); resolver.assert_calls(&[(&expected_key, "aud-transformed")]); } diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs new file mode 100644 index 0000000000..1d9c36a509 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-owned request traces with untrusted workload links. + +use opentelemetry::propagation::{Extractor, TextMapPropagator as _}; +use opentelemetry::trace::{SpanContext, TraceContextExt as _}; +use opentelemetry::{Context, KeyValue}; +use opentelemetry_sdk::propagation::TraceContextPropagator; +use tracing::Span; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; + +use super::correlation::HttpOutcome; + +const MAX_TRACEPARENT_BYTES: usize = 256; + +#[derive(Debug)] +pub(super) struct RequestTrace { + span: Span, + outcome: &'static str, + failure_reason: Option<&'static str>, +} + +#[derive(Debug, Clone, Copy)] +pub enum RequestStage { + PolicyEvaluate, + RequestMiddleware, + WebsocketPreflight, + CredentialPrepare, + UpstreamResolve, + UpstreamDial, + UpstreamSend, + Response, + ResponseMiddleware, +} + +impl RequestStage { + fn name(self) -> &'static str { + match self { + Self::PolicyEvaluate => "openshell.policy.evaluate", + Self::RequestMiddleware => "openshell.middleware.request", + Self::WebsocketPreflight => "openshell.middleware.websocket.preflight", + Self::CredentialPrepare => "openshell.credential.prepare", + Self::UpstreamResolve => "openshell.upstream.resolve", + Self::UpstreamDial => "openshell.upstream.connect", + Self::UpstreamSend => "openshell.upstream.send", + Self::Response => "openshell.http.response", + Self::ResponseMiddleware => "openshell.middleware.response", + } + } +} + +impl RequestTrace { + pub(super) fn new( + request_id: &str, + method: &str, + host: &str, + port: u16, + raw_request: &[u8], + ) -> Self { + let span = tracing::info_span!( + parent: None, + "openshell.http.request", + openshell.request.id = request_id, + http.request.method = normalized_method(method), + server.address = host, + server.port = i64::from(port), + openshell.request.outcome = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + http.response.status_code = tracing::field::Empty, + otel.kind = "server", + otel.status_code = tracing::field::Empty, + ); + if let Some(context) = workload_link(raw_request) { + span.add_link_with_attributes( + context, + vec![ + KeyValue::new("openshell.correlation.source", "http.traceparent"), + KeyValue::new("openshell.correlation.trust", "workload"), + ], + ); + } + Self { + span, + outcome: "incomplete", + failure_reason: None, + } + } + + pub(super) fn span(&self) -> Span { + self.span.clone() + } + + pub(super) fn stage(&self, stage: RequestStage) -> Span { + if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() { + return Span::none(); + } + tracing::info_span!( + parent: &self.span, + "openshell.http.stage", + otel.name = stage.name(), + otel.status_code = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + ) + } + + pub(super) fn record_status(&self, status: u16) { + self.span + .record("http.response.status_code", i64::from(status)); + } + + pub(super) fn record_outcome(&mut self, outcome: HttpOutcome) { + let outcome = match outcome { + _ if self.outcome == "failed" => "failed", + HttpOutcome::Allowed if self.outcome == "denied" => "denied", + HttpOutcome::Allowed => "allowed", + HttpOutcome::Denied => "denied", + HttpOutcome::PlatformError | HttpOutcome::UpstreamError => "failed", + HttpOutcome::Upgraded => "upgraded", + }; + self.outcome = outcome; + self.span.record("openshell.request.outcome", outcome); + } + + pub(super) fn record_failure_reason(&mut self, reason: &'static str) { + if self.failure_reason.is_none() { + self.failure_reason = Some(reason); + self.span.record("openshell.request.failure_reason", reason); + } + } + + pub(super) fn record_unclassified_failure(&mut self) { + self.record_outcome(HttpOutcome::PlatformError); + self.record_failure_reason("request_failed"); + } +} + +impl Drop for RequestTrace { + fn drop(&mut self) { + // Finalize exported timing after the request's child stages. + self.span.in_scope(|| { + self.span.record("openshell.request.outcome", self.outcome); + if matches!(self.outcome, "failed" | "incomplete") { + self.span.record("otel.status_code", "ERROR"); + } + }); + } +} + +fn normalized_method(method: &str) -> &'static str { + match method { + "GET" => "GET", + "HEAD" => "HEAD", + "POST" => "POST", + "PUT" => "PUT", + "DELETE" => "DELETE", + "CONNECT" => "CONNECT", + "OPTIONS" => "OPTIONS", + "TRACE" => "TRACE", + "PATCH" => "PATCH", + _ => "_OTHER", + } +} + +fn workload_link(raw_request: &[u8]) -> Option { + let end = raw_request + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n")?; + let mut value = None; + for line in raw_request[..end].split(|byte| *byte == b'\n').skip(1) { + let line = line.strip_suffix(b"\r").unwrap_or(line); + let Some(colon) = line.iter().position(|byte| *byte == b':') else { + continue; + }; + if !line[..colon].eq_ignore_ascii_case(b"traceparent") { + continue; + } + if value.is_some() { + return None; + } + let candidate = std::str::from_utf8(&line[colon + 1..]).ok()?.trim(); + if candidate.len() > MAX_TRACEPARENT_BYTES { + return None; + } + value = Some(candidate); + } + let context = TraceContextPropagator::new() + .extract_with_context(&Context::new(), &OnlyTraceparent(value?)); + let span_context = context.span().span_context().clone(); + span_context.is_valid().then_some(span_context) +} + +struct OnlyTraceparent<'a>(&'a str); + +impl Extractor for OnlyTraceparent<'_> { + fn get(&self, key: &str) -> Option<&str> { + (key == "traceparent").then_some(self.0) + } + + fn keys(&self) -> Vec<&str> { + vec!["traceparent"] + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs new file mode 100644 index 0000000000..c4ef2e6d04 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::l7::correlation::HttpOutcome; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::{InMemorySpanExporter, Sampler, SdkTracerProvider, SpanData}; +use tracing_subscriber::layer::SubscriberExt as _; + +const TRACE_ID: &str = "4bf92f3577b34da6a3ce929d0e0e4736"; +const PARENT_ID: &str = "00f067aa0ba902b7"; + +fn request(flags: &str) -> Vec { + format!( + "GET /account?secret=DO-NOT-RECORD HTTP/1.1\r\nHost: example.test\r\ntraceparent: 00-{TRACE_ID}-{PARENT_ID}-{flags}\r\ntracestate: vendor=DO-NOT-RECORD\r\nbaggage: secret=DO-NOT-RECORD\r\n\r\n" + ) + .into_bytes() +} + +fn capture(sampler: Sampler, emit: impl FnOnce()) -> Vec { + let capture = RequestTracing::new(Some(sampler)); + tracing::subscriber::with_default(capture.subscriber(), emit); + let spans = capture.spans(); + capture.shutdown(); + spans +} + +fn root(spans: &[SpanData]) -> &SpanData { + spans + .iter() + .find(|span| span.name == "openshell.http.request") + .expect("one supervisor request root") +} + +#[test] +fn workload_context_is_linked_not_parented_for_both_sampled_flags() { + for flags in ["00", "01"] { + let spans = capture(Sampler::ParentBased(Box::new(Sampler::AlwaysOn)), || { + let unrelated = tracing::info_span!("unrelated-host-operation"); + unrelated.in_scope(|| { + let mut trace = RequestTrace::new( + "test-request-id", + "GET", + "example.test", + 443, + &request(flags), + ); + trace.stage(RequestStage::PolicyEvaluate).in_scope(|| {}); + trace.record_status(403); + trace.record_outcome(HttpOutcome::Denied); + }); + }); + let root = root(&spans); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(root.links.links.len(), 1); + let link = &root.links.links[0]; + assert_eq!(link.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(link.span_context.span_id().to_string(), PARENT_ID); + assert_eq!(link.span_context.is_sampled(), flags == "01"); + assert!(link.span_context.trace_state().header().is_empty()); + assert!(link.attributes.iter().any(|field| { + field.key.as_str() == "openshell.correlation.trust" + && field.value.as_str() == "workload" + })); + let policy = spans + .iter() + .find(|span| span.name == "openshell.policy.evaluate") + .unwrap(); + assert_eq!(policy.parent_span_id, root.span_context.span_id()); + assert_eq!(policy.span_context.trace_id(), root.span_context.trace_id()); + assert_eq!( + span_attribute(root, "openshell.request.id") + .unwrap() + .as_str(), + "test-request-id" + ); + assert_eq!(root.status, opentelemetry::trace::Status::Unset); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD")); + } +} + +#[test] +fn explicit_request_root_does_not_adopt_ambient_otel_context() { + let remote = workload_link(&request("01")).unwrap(); + let spans = capture(Sampler::AlwaysOn, || { + let _guard = Context::new().with_remote_span_context(remote).attach(); + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("00")); + trace.record_outcome(HttpOutcome::Allowed); + }); + let root = root(&spans); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(root.links.links.len(), 1); +} + +#[test] +fn failures_cannot_be_overwritten_by_later_success_or_decisions() { + for failure in [HttpOutcome::PlatformError, HttpOutcome::UpstreamError] { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + trace.record_outcome(failure); + trace.record_failure_reason("upstream_connect_failed"); + trace.record_outcome(HttpOutcome::Allowed); + trace.record_outcome(HttpOutcome::Denied); + trace.record_outcome(HttpOutcome::Upgraded); + trace.record_failure_reason("request_failed"); + }); + let root = root(&spans); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + } +} + +#[test] +fn request_completion_encloses_stages_after_an_early_root_exit() { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + trace.span().in_scope(|| trace.record_status(200)); + trace.stage(RequestStage::Response).in_scope(|| {}); + trace.record_outcome(HttpOutcome::Allowed); + }); + let root = root(&spans); + let response = spans + .iter() + .find(|span| span.name == RequestStage::Response.name()) + .unwrap(); + assert_eq!(response.parent_span_id, root.span_context.span_id()); + assert!(response.start_time >= root.start_time); + assert!(response.end_time <= root.end_time); +} + +#[test] +fn remote_sampled_flag_cannot_override_local_always_off() { + for flags in ["00", "01"] { + let spans = capture(Sampler::ParentBased(Box::new(Sampler::AlwaysOff)), || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request(flags)); + trace.stage(RequestStage::PolicyEvaluate).in_scope(|| {}); + trace.record_outcome(HttpOutcome::Denied); + }); + assert!(spans.is_empty()); + } +} + +#[test] +fn every_outcome_has_the_same_root_shape_and_stages_have_fixed_names() { + let stages = [ + RequestStage::PolicyEvaluate, + RequestStage::RequestMiddleware, + RequestStage::WebsocketPreflight, + RequestStage::CredentialPrepare, + RequestStage::UpstreamResolve, + RequestStage::UpstreamDial, + RequestStage::UpstreamSend, + RequestStage::Response, + RequestStage::ResponseMiddleware, + ]; + for (outcome, failed) in [ + (HttpOutcome::Allowed, false), + (HttpOutcome::Denied, false), + (HttpOutcome::UpstreamError, true), + ] { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + for stage in stages { + trace.stage(stage).in_scope(|| {}); + } + trace.record_outcome(outcome); + }); + let root = root(&spans); + assert_eq!(spans.len(), stages.len() + 1); + for stage in stages { + let child = spans.iter().find(|span| span.name == stage.name()).unwrap(); + assert_eq!(child.parent_span_id, root.span_context.span_id()); + assert_eq!(child.span_context.trace_id(), root.span_context.trace_id()); + } + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + failed + ); + } +} + +#[test] +fn malformed_duplicate_oversized_and_zero_context_is_ignored() { + let valid = String::from_utf8(request("01")).unwrap(); + let duplicate = valid.replace( + "\r\ntracestate:", + &format!("\r\nTraceparent: 00-{TRACE_ID}-{PARENT_ID}-00\r\ntracestate:"), + ); + for headers in [ + b"GET / HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + b"GET / HTTP/1.1\r\ntraceparent: invalid\r\n\r\n".to_vec(), + duplicate.into_bytes(), + valid + .replace(TRACE_ID, "00000000000000000000000000000000") + .into_bytes(), + valid.replace(PARENT_ID, "0000000000000000").into_bytes(), + format!("GET / HTTP/1.1\r\ntraceparent: {}\r\n\r\n", "a".repeat(257)).into_bytes(), + ] { + assert!(workload_link(&headers).is_none()); + } +} + +#[test] +fn unrelated_opaque_header_values_do_not_discard_valid_trace_context() { + let mut headers = b"GET / HTTP/1.1\r\nX-Opaque: \xff\r\n".to_vec(); + headers + .extend_from_slice(format!("traceparent: 00-{TRACE_ID}-{PARENT_ID}-01\r\n\r\n").as_bytes()); + assert!(workload_link(&headers).is_some()); +} + +#[test] +fn custom_method_values_are_not_exported() { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new( + "local", + "DO-NOT-RECORD", + "example.test", + 443, + &request("01"), + ); + trace.record_outcome(HttpOutcome::Allowed); + }); + assert_eq!( + span_attribute(root(&spans), "http.request.method") + .unwrap() + .as_str(), + "_OTHER" + ); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD")); +} + +#[test] +fn a_filtered_request_root_does_not_create_orphan_stage_spans() { + use tracing_subscriber::Layer as _; + + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture).with( + tracing_opentelemetry::layer() + .with_tracer(provider.tracer("filtered-root-test")) + .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { + metadata.name() != "openshell.http.request" + })), + ); + tracing::subscriber::with_default(subscriber, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + assert!( + !trace.span().is_disabled(), + "another layer still enables the root" + ); + assert!(trace.stage(RequestStage::PolicyEvaluate).is_disabled()); + trace.record_outcome(HttpOutcome::Allowed); + }); + provider.force_flush().unwrap(); + assert!(exporter.get_finished_spans().unwrap().is_empty()); + provider.shutdown().unwrap(); +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 8bc198b9ee..caec858eaf 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,6 +27,7 @@ pub(crate) use request_body::{ }; use crate::l7::EndpointObserver; +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -44,6 +45,7 @@ use openshell_core::secrets::{ }; use openshell_ocsf::ctx::ctx as ocsf_ctx; use sha1::{Digest, Sha1}; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt::{self, Write as _}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; @@ -164,6 +166,7 @@ pub(crate) struct DenyResponseContext<'a> { pub(crate) port: Option, pub(crate) binary: Option<&'a str>, pub(crate) agent_proposals_enabled: bool, + pub(crate) request_id: Option<&'a str>, } impl<'a> DenyResponseContext<'a> { @@ -174,11 +177,25 @@ impl<'a> DenyResponseContext<'a> { port: Some(ctx.port), binary: Some(&ctx.binary_path), agent_proposals_enabled: ctx.agent_proposals.enabled(), + request_id: None, } } + + #[must_use] + pub(crate) fn with_request_id(mut self, request_id: &'a str) -> Self { + self.request_id = Some(request_id); + self + } } impl RestProvider { + pub(crate) async fn parse_uninspected_request( + &self, + client: &mut C, + ) -> Result> { + parse_http_request_with_receipt_protection(client, &self.canonicalize_options, false).await + } + /// Deny with a redacted target for the response body. pub(crate) async fn deny_with_redacted_target( &self, @@ -204,6 +221,14 @@ impl RestProvider { async fn parse_http_request( client: &mut C, canonicalize_options: &crate::l7::path::CanonicalizeOptions, +) -> Result> { + parse_http_request_with_receipt_protection(client, canonicalize_options, true).await +} + +async fn parse_http_request_with_receipt_protection( + client: &mut C, + canonicalize_options: &crate::l7::path::CanonicalizeOptions, + protect_receipt: bool, ) -> Result> { let mut buf = Vec::with_capacity(4096); @@ -308,6 +333,10 @@ async fn parse_http_request( )?; } + if protect_receipt && let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&buf) { + buf = sanitized; + } + Ok(Some(L7Request { action: method, target: canonical.path, @@ -327,6 +356,20 @@ pub(crate) fn request_from_buffered_http( target: impl Into, query_target: &str, raw_header: Vec, +) -> Result { + let mut request = + request_from_buffered_http_unprotected(action, target, query_target, raw_header)?; + if let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&request.raw_header) { + request.raw_header = sanitized; + } + Ok(request) +} + +pub(crate) fn request_from_buffered_http_unprotected( + action: impl Into, + target: impl Into, + query_target: &str, + raw_header: Vec, ) -> Result { let header_end = raw_header .windows(4) @@ -897,7 +940,7 @@ where U: AsyncRead + AsyncWrite + Unpin, { relay_http_request_with_response_middleware_guarded_observed( - req, client, upstream, options, None, None, + req, client, upstream, options, None, None, None, ) .await } @@ -909,12 +952,13 @@ pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed options: RelayRequestOptions<'_>, response_middleware: Option>, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - Box::pin(relay_http_request_with_body_guarded_observed( + Box::pin(relay_http_request_with_body_guarded_observed_correlated( req, client, upstream, @@ -922,13 +966,15 @@ where None, response_middleware, observer, + correlation, )) .await } /// Relay one request and its response. `live_body` carries a body that HTTP /// hook request middleware streams; `req` then holds only its head. -pub(crate) async fn relay_http_request_with_body_guarded_observed( +#[allow(clippy::too_many_arguments)] +pub(crate) async fn relay_http_request_with_body_guarded_observed_correlated( req: &L7Request, client: &mut C, upstream: &mut U, @@ -936,12 +982,19 @@ pub(crate) async fn relay_http_request_with_body_guarded_observed( live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, response_middleware: Option>, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - let mut observed_upstream = ObservedUpstream { upstream, observer }; + let mut observed_upstream = ObservedUpstream { + upstream, + observer, + correlation, + request_span: None, + track_request_span: live_body.is_none(), + }; let upstream = &mut observed_upstream; let mut options = options; if let Some(middleware) = response_middleware.as_ref() { @@ -978,6 +1031,7 @@ where websocket: None, client_requested_upgrade, observer, + correlation, }, response_middleware, ) @@ -1025,8 +1079,32 @@ where offered_subprotocols: request.subprotocols.clone(), }); + let rewrite_span = + if options.resolver.is_some() || contains_reserved_credential_marker_bytes(&header_bytes) { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }) + } else { + tracing::Span::none() + }; let rewrite_result = - rewrite_http_header_block(&header_bytes, options.resolver).map_err(miette::Report::new)?; + rewrite_span.in_scope(|| rewrite_http_header_block(&header_bytes, options.resolver)); + rewrite_span.record( + "otel.status_code", + if rewrite_result.is_ok() { + "OK" + } else { + "ERROR" + }, + ); + if rewrite_result.is_err() { + rewrite_span.record( + "openshell.request.failure_reason", + "credential_resolution_refused", + ); + } + drop(rewrite_span); + let rewrite_result = rewrite_result.map_err(miette::Report::new)?; if let Some(validation) = options.mcp_request_validation { // Header credential resolution and hop-by-hop cleanup must finish @@ -1048,6 +1126,7 @@ where validation.ctx, validation.redacted_target, observer, + correlation, ) .await? { @@ -1180,35 +1259,55 @@ where guard.ensure_current()?; } - let signed = crate::sigv4::apply_sigv4_to_request( - &full_request, - options.host, - ®ion, - service, - access_key, - secret_key, - session_token, - )?; + let sign = || { + crate::sigv4::apply_sigv4_to_request( + &full_request, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + ) + }; + let signed = correlation.map_or_else(sign, |correlation| { + correlation.run_stage_sync( + RequestStage::CredentialPrepare, + HttpOutcome::PlatformError, + "credential_signing_failed", + sign, + ) + })?; ensure_credential_generation_current(options)?; upstream.write_all(&signed).await.into_diagnostic()?; } else { // Sign headers only, stream body through. - let signable_body = match payload_mode { - SigV4PayloadMode::StreamingUnsignedTrailer => { - SignableBody::StreamingUnsignedPayloadTrailer - } - _ => SignableBody::UnsignedPayload, + let sign = || { + let signable_body = match payload_mode { + SigV4PayloadMode::StreamingUnsignedTrailer => { + SignableBody::StreamingUnsignedPayloadTrailer + } + _ => SignableBody::UnsignedPayload, + }; + crate::sigv4::apply_sigv4_headers_only_with_body( + &rewrite_result.rewritten, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + signable_body, + ) }; - let signed_headers = crate::sigv4::apply_sigv4_headers_only_with_body( - &rewrite_result.rewritten, - options.host, - ®ion, - service, - access_key, - secret_key, - session_token, - signable_body, - )?; + let signed_headers = correlation.map_or_else(sign, |correlation| { + correlation.run_stage_sync( + RequestStage::CredentialPrepare, + HttpOutcome::PlatformError, + "credential_signing_failed", + sign, + ) + })?; ensure_credential_generation_current(options)?; upstream .write_all(&signed_headers) @@ -1234,8 +1333,14 @@ where } } BodyLength::Chunked => { - relay_chunked(client, upstream, overflow, options.generation_guard) - .await?; + relay_chunked_with_receipt_protection( + client, + upstream, + overflow, + options.generation_guard, + correlation.is_some(), + ) + .await?; } BodyLength::None => { if !overflow.is_empty() { @@ -1301,6 +1406,7 @@ where &rewrite_result.rewritten, &req.raw_header[header_end..], options, + correlation.is_some(), ) .await { @@ -1329,7 +1435,14 @@ where } } BodyLength::Chunked => { - relay_chunked(client, upstream, overflow, options.generation_guard).await?; + relay_chunked_with_receipt_protection( + client, + upstream, + overflow, + options.generation_guard, + correlation.is_some(), + ) + .await?; } BodyLength::None => { if !overflow.is_empty() { @@ -1339,8 +1452,9 @@ where } } upstream.flush().await.into_diagnostic()?; + upstream.request_span.take(); - let outcome = relay_response( + let response = relay_response( &req.action, upstream, client, @@ -1349,10 +1463,21 @@ where websocket: websocket_response, client_requested_upgrade, observer, + correlation, }, response_middleware, - ) - .await?; + ); + let outcome = if let Some(correlation) = correlation { + Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + response, + )) + .await? + } else { + response.await? + }; Ok(outcome) } @@ -1376,6 +1501,21 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() struct ObservedUpstream<'a, U> { upstream: &'a mut U, observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, + request_span: Option, + track_request_span: bool, +} + +impl ObservedUpstream<'_, U> { + fn record_write_failure(&self) { + if let Some(span) = &self.request_span { + span.record("otel.status_code", "ERROR"); + span.record("openshell.request.failure_reason", "upstream_send_failed"); + } + if let Some(correlation) = self.correlation { + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); + } + } } impl AsyncRead for ObservedUpstream<'_, U> { @@ -1402,11 +1542,22 @@ impl AsyncWrite for ObservedUpstream<'_, U> { buf: &[u8], ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf); - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer - { - observer.observe(EndpointResult::TransportFailed); + let result = if this.track_request_span { + let span = this.request_span.get_or_insert_with(|| { + this.correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::UpstreamSend) + }) + }); + span.in_scope(|| std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf)) + } else { + std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf) + }; + if matches!(result, std::task::Poll::Ready(Err(_))) { + this.record_write_failure(); + if let Some(observer) = this.observer { + observer.observe(EndpointResult::TransportFailed); + } } result } @@ -1416,11 +1567,20 @@ impl AsyncWrite for ObservedUpstream<'_, U> { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.upstream).poll_flush(cx); - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer + let result = if let Some(span) = &this.request_span { + span.in_scope(|| std::pin::Pin::new(&mut *this.upstream).poll_flush(cx)) + } else { + std::pin::Pin::new(&mut *this.upstream).poll_flush(cx) + }; + if matches!(result, std::task::Poll::Ready(Err(_))) { + this.record_write_failure(); + if let Some(observer) = this.observer { + observer.observe(EndpointResult::TransportFailed); + } + } else if matches!(result, std::task::Poll::Ready(Ok(()))) + && let Some(span) = &this.request_span { - observer.observe(EndpointResult::TransportFailed); + span.record("otel.status_code", "OK"); } result } @@ -1468,6 +1628,7 @@ async fn relay_request_body_with_marker_guard( headers: &[u8], already_read: &[u8], options: RelayRequestOptions<'_>, + protect_receipt: bool, ) -> Result<()> where C: AsyncRead + Unpin, @@ -1509,7 +1670,14 @@ where write_body_bytes(upstream, &scanner.finish()?, options).await?; } BodyLength::Chunked => { - relay_chunked_with_marker_guard(client, upstream, already_read, options).await?; + relay_chunked_with_marker_guard( + client, + upstream, + already_read, + options, + protect_receipt, + ) + .await?; } } Ok(()) @@ -1520,6 +1688,7 @@ async fn relay_chunked_with_marker_guard( upstream: &mut U, already_read: &[u8], options: RelayRequestOptions<'_>, + protect_receipt: bool, ) -> Result<()> where C: AsyncRead + Unpin, @@ -1551,11 +1720,19 @@ where if contains_reserved_credential_marker_bytes(&trailer) { return Err(BodyCredentialError::Trailer.into()); } - write_body_bytes(upstream, &trailer, options).await?; - write_body_bytes(upstream, b"\r\n", options).await?; if trailer.is_empty() { + write_body_bytes(upstream, b"\r\n", options).await?; return Ok(()); } + if !protect_receipt + || !http_field_line_has_name( + &trailer, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + { + write_body_bytes(upstream, &trailer, options).await?; + write_body_bytes(upstream, b"\r\n", options).await?; + } } } @@ -2371,6 +2548,102 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { Ok(out.into_bytes()) } +fn strip_platform_request_id_from_head(headers: &[u8]) -> Cow<'_, [u8]> { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut out: Option> = None; + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .windows(2) + .position(|window| window == b"\r\n") + .map_or(header_block.len(), |end| position + end); + let line = &header_block[position..line_end]; + if line.is_empty() { + if position + 2 < header_block.len() { + out.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(out) = &mut out { + out.extend_from_slice(b"\r\n"); + } + break; + } + if let Some(colon) = line.iter().position(|byte| *byte == b':') { + let name = trim_ascii_ows(&line[..colon]); + let receipt = name.eq_ignore_ascii_case(protected); + let trailer = name.eq_ignore_ascii_case(b"trailer") + && line[colon + 1..] + .split(|byte| *byte == b',') + .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)); + if receipt || trailer { + let out = out.get_or_insert_with(|| { + let mut out = Vec::with_capacity(headers.len()); + out.extend_from_slice(&header_block[..position]); + out + }); + if trailer { + let mut values = line[colon + 1..] + .split(|byte| *byte == b',') + .filter(|value| !trim_ascii_ows(value).eq_ignore_ascii_case(protected)); + let mut retained = values.clone(); + if retained.any(|value| !trim_ascii_ows(value).is_empty()) { + out.extend_from_slice(&line[..=colon]); + if let Some(first) = values.next() { + out.extend_from_slice(first); + } + for value in values { + out.push(b','); + out.extend_from_slice(value); + } + out.extend_from_slice(b"\r\n"); + } + } + position = line_end.saturating_add(2); + continue; + } + } + if line_end + 2 > header_block.len() { + out.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(out) = &mut out { + out.extend_from_slice(line); + out.extend_from_slice(b"\r\n"); + } + position = line_end.saturating_add(2); + } + out.map_or(Cow::Borrowed(headers), |mut out| { + out.extend_from_slice(buffered_suffix); + Cow::Owned(out) + }) +} + +fn trim_ascii_ows(mut value: &[u8]) -> &[u8] { + while value + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[1..]; + } + while value + .last() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[..value.len() - 1]; + } + value +} + +fn http_field_line_has_name(line: &[u8], expected: &str) -> bool { + line.iter() + .position(|byte| *byte == b':') + .is_some_and(|end| line[..end].eq_ignore_ascii_case(expected.as_bytes())) +} + pub(crate) fn connection_nominated_header_names(headers: &[u8]) -> Result> { validate_http_request_header_block(headers)?; let header_str = @@ -2906,7 +3179,15 @@ async fn send_deny_response( context: Option>, ) -> Result<()> { let body = deny_response_body(req, policy_name, reason, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body, + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a middleware-specific 403 response without policy-advisor guidance. @@ -2922,7 +3203,15 @@ pub(crate) async fn send_middleware_deny_response( context: Option>, ) -> Result<()> { let body = middleware_deny_response_body(req, policy_name, denial, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body, + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned response for fail-closed middleware failures. @@ -2943,7 +3232,15 @@ pub(crate) async fn send_middleware_failure_response( if let Some(failure) = failure { insert_middleware_failure(&mut body, failure); } - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body, + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Add a middleware failure's stage and platform-owned code to a response @@ -2981,7 +3278,15 @@ pub(crate) async fn send_middleware_unavailable_response( code: openshell_supervisor_middleware::HttpFailureCode::Unavailable, }, ); - send_json_response(policy_name, body, client, "503 Service Unavailable").await + send_json_response_internal( + policy_name, + body, + client, + "503 Service Unavailable", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned 408 when the sandbox stops sending a request body @@ -3001,15 +3306,15 @@ pub(crate) async fn send_request_timeout_response( serde_json::json!("Request body was not received in time"), ); } - send_json_response(policy_name, body, client, "408 Request Timeout").await -} - -async fn send_forbidden_json( - policy_name: &str, - body: serde_json::Value, - client: &mut C, -) -> Result<()> { - send_json_response(policy_name, body, client, "403 Forbidden").await + send_json_response_internal( + policy_name, + body, + client, + "408 Request Timeout", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned JSON response with an explicit HTTP status. @@ -3030,15 +3335,70 @@ pub(crate) async fn send_json_response_with_allow( client: &mut C, status: &str, allowed_methods: Option<&'static str>, +) -> Result<()> { + send_json_response_internal(policy_name, body, client, status, allowed_methods, None).await +} + +pub(crate) async fn send_json_response_with_allow_and_request_id( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: &str, +) -> Result<()> { + send_json_response_internal( + policy_name, + body, + client, + status, + allowed_methods, + Some(request_id), + ) + .await +} + +pub(crate) async fn send_json_response_with_request_id( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + request_id: &str, +) -> Result<()> { + send_json_response_with_allow_and_request_id( + policy_name, + body, + client, + status, + None, + request_id, + ) + .await +} + +async fn send_json_response_internal( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: Option<&str>, ) -> Result<()> { let body_bytes = body.to_string(); let allow = allowed_methods.map_or_else(String::new, |methods| format!("Allow: {methods}\r\n")); + let request_id = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); let response = format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ X-OpenShell-Policy: {}\r\n\ {allow}\ + {request_id}\ Connection: close\r\n\ \r\n\ {}", @@ -3365,11 +3725,12 @@ where /// `already_read` are overflow bytes buffered while parsing the headers. They /// are validated here before being written so ambiguous framing cannot reach /// the peer. -async fn relay_chunked( +async fn relay_chunked_with_receipt_protection( reader: &mut R, writer: &mut W, already_read: &[u8], generation_guard: Option<&PolicyGenerationGuard>, + protect_receipt: bool, ) -> Result<()> where R: AsyncRead + Unpin, @@ -3394,6 +3755,7 @@ where let mut trailer_count = 0usize; let mut trailer_bytes = 0usize; loop { + let pending_start = input.pending_len(); let trailer_line = input .read_line( reader, @@ -3419,6 +3781,14 @@ where ); return Ok(()); } + if protect_receipt + && http_field_line_has_name( + &trailer_line, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + { + input.pending.truncate(pending_start); + } if trailer_count == MAX_CHUNK_TRAILER_FIELDS { return Err(miette!( "Chunk trailers exceed {MAX_CHUNK_TRAILER_FIELDS} fields" @@ -3906,7 +4276,7 @@ mod tests { const TEXT_OPCODE: u8 = 0x1; #[derive(Debug, Clone, Copy)] - enum ResponseRelayScript { + pub(super) enum ResponseRelayScript { HeadersOnly, WholeBody, WholeBodyWithTrailer, @@ -4243,9 +4613,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an incomplete chunk must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an incomplete chunk must fail"); assert!(error.to_string().contains("ended mid-chunk")); assert_eq!( @@ -4262,9 +4633,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an oversized chunk-size line must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an oversized chunk-size line must fail"); assert!(error.to_string().contains("Chunk-size line exceeds")); } @@ -4281,9 +4653,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an oversized trailer block must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an oversized trailer block must fail"); assert!(error.to_string().contains("Chunk trailers exceed")); } @@ -4298,9 +4671,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("too many trailer fields must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("too many trailer fields must fail"); assert!( error @@ -4326,9 +4700,15 @@ mod tests { let mut reader = CountingReader::new(input); let mut writer = CountingWriter::default(); - let error = relay_chunked(&mut reader, &mut writer, already_read, None) - .await - .expect_err("ambiguous chunked framing must fail closed"); + let error = relay_chunked_with_receipt_protection( + &mut reader, + &mut writer, + already_read, + None, + false, + ) + .await + .expect_err("ambiguous chunked framing must fail closed"); assert!( error.to_string().contains("chunk"), @@ -4774,6 +5154,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4838,6 +5219,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: false, + request_id: None, }), ); @@ -4941,6 +5323,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/curl"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4979,6 +5362,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: Some("request-123"), }), ) .await @@ -4993,12 +5377,14 @@ mod tests { assert!(response.starts_with("HTTP/1.1 403 Forbidden")); assert!(response.contains("Content-Type: application/json")); assert!(response.contains("X-OpenShell-Policy: github-readonly")); + assert!(response.contains("x-openshell-request-id: request-123")); let (_, body) = response.split_once("\r\n\r\n").unwrap(); let body: serde_json::Value = serde_json::from_str(body).unwrap(); assert_eq!(body["error"], "policy_denied"); assert_eq!(body["method"], "POST"); assert_eq!(body["path"], "/user/repos"); + assert!(body.get("request_id").is_none()); assert_eq!(body["rule_missing"]["host"], "api.github.com"); assert_eq!(body["next_steps"][2]["action"], "inspect_recent_denials"); assert!(body["agent_guidance"].as_str().unwrap().contains("retry")); @@ -5104,9 +5490,15 @@ mod tests { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); - relay_chunked(&mut relay_reader, &mut relay_writer, &[], None) - .await - .expect("chunked body should relay"); + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + &[], + None, + false, + ) + .await + .expect("chunked body should relay"); let mut remaining = vec![0; pipelined_request.len()]; relay_reader.read_exact(&mut remaining).await.unwrap(); @@ -5132,9 +5524,15 @@ mod tests { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); - relay_chunked(&mut relay_reader, &mut relay_writer, already_read, None) - .await - .expect("chunked body with trailers should relay"); + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + already_read, + None, + false, + ) + .await + .expect("chunked body with trailers should relay"); let mut remaining = vec![0; pipelined_request.len()]; relay_reader.read_exact(&mut remaining).await.unwrap(); @@ -5148,6 +5546,38 @@ mod tests { assert_eq!(forwarded, expected); } + #[tokio::test] + async fn relay_chunked_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nX-OpenShell-Request-ID: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, true) + .await + .expect("chunked body should relay"); + + assert_eq!(writer, b"3\r\nabc\r\n0\r\nX-Checksum: ok\r\n\r\n"); + } + + #[tokio::test] + async fn guarded_chunked_request_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nx-openshell-request-id: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked_with_marker_guard( + &mut reader, + &mut writer, + &[], + RelayRequestOptions::default(), + true, + ) + .await + .expect("guarded chunked body should relay"); + + assert_eq!(writer, b"3\r\nabc\r\n0\r\nX-Checksum: ok\r\n\r\n"); + } + #[tokio::test] async fn relay_chunked_handles_tiny_chunks_without_an_aggregate_framing_limit() { let mut chunked_body = Vec::new(); @@ -5162,7 +5592,7 @@ mod tests { let mut reader = tokio::io::BufReader::with_capacity(RELAY_BUF_SIZE, CountingReader::new(wire)); let mut writer = CountingWriter::default(); - relay_chunked(&mut reader, &mut writer, &[], None) + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) .await .expect("valid tiny chunks must not be rejected by an aggregate framing limit"); @@ -5190,7 +5620,14 @@ mod tests { let relay = tokio::spawn(async move { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let mut relay_writer = relay_writer; - relay_chunked(&mut relay_reader, &mut relay_writer, &[], None).await + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + &[], + None, + false, + ) + .await }); let first_chunk = b"5\r\nhello\r\n"; @@ -5405,6 +5842,18 @@ mod tests { assert!(parse_body_length(headers).is_err()); } + #[test] + fn platform_request_id_is_stripped_without_touching_body() { + let suffix = b"4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged-trailer\r\n\r\n"; + let mut request = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nX-OpenShell-Request-ID: forged-head\r\nTrailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n".to_vec(); + request.extend_from_slice(suffix); + + let stripped = strip_platform_request_id_from_head(&request); + let mut expected = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nTrailer: X-Checksum\r\n\r\n".to_vec(); + expected.extend_from_slice(suffix); + assert_eq!(stripped, expected); + } + #[test] fn strip_connection_nominated_headers_before_forwarding() { let raw = b"GET /api HTTP/1.1\r\nHost: x\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; @@ -6409,6 +6858,7 @@ mod tests { }, None, Some(&observer), + None, ), ) .await @@ -6642,6 +7092,9 @@ mod tests { let mut upstream = ObservedUpstream { upstream: &mut response, observer: Some(&observer), + correlation: None, + request_span: None, + track_request_span: true, }; let mut client = Vec::new(); relay_response( @@ -6953,7 +7406,7 @@ mod tests { } } - fn response_middleware_fixture( + pub(super) fn response_middleware_fixture( script: ResponseRelayScript, ) -> ( openshell_supervisor_middleware::ChainRunner, @@ -6989,7 +7442,7 @@ mod tests { (runner, chain) } - fn response_middleware_context<'a>( + pub(super) fn response_middleware_context<'a>( runner: &'a openshell_supervisor_middleware::ChainRunner, chain: &'a [openshell_supervisor_middleware::ChainEntry], method: &str, @@ -8644,7 +9097,13 @@ mod tests { failure_category: Some("timeout".into()), }], ); - let json = events[0].to_json().unwrap().to_string(); + let event = events[0].to_json().unwrap(); + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + let json = event.to_string(); for forbidden in [ "secret-response-body", "authorization", @@ -10895,6 +11354,7 @@ mod tests { )), ..Default::default() }, + false, ) .await .unwrap(); @@ -11040,6 +11500,7 @@ mod tests { &mut upstream_writer, wire, RelayRequestOptions::default(), + false, ) .await .expect_err("encoded placeholder must fail closed"); @@ -11060,6 +11521,7 @@ mod tests { &mut output, wire, RelayRequestOptions::default(), + false, ) .await .expect_err("ambiguous trailer framing must fail closed"); @@ -11425,7 +11887,12 @@ mod tests { } #[tokio::test] - async fn relay_sigv4_body_signing_signs_empty_body_without_content_length() { + async fn relay_sigv4_signing_records_success_and_failure() { + use crate::test_support::request_tracing::{RequestTracing, span_attribute}; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + + let _lock = openshell_otel_test_support::tracing_test_lock().await; let (_, resolver) = SecretResolver::from_provider_env( [ ("AWS_ACCESS_KEY_ID".to_string(), "AKIATESTKEY".to_string()), @@ -11448,57 +11915,83 @@ mod tests { raw_header, body_length: BodyLength::None, }; - let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); - let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); - - let upstream_task = tokio::spawn(async move { - let mut buf = vec![0u8; 8192]; - let mut total = 0usize; - loop { - let n = upstream_side.read(&mut buf[total..]).await.unwrap(); - if n == 0 { - break; + for signing in [ + crate::l7::CredentialSigning::SigV4Body, + crate::l7::CredentialSigning::SigV4NoBody, + ] { + for host in ["s3.us-east-1.amazonaws.com", "invalid host"] { + let succeeds = host != "invalid host"; + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); + let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); + upstream_side + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { + let ctx = crate::l7::relay::L7EvalContext { + host: host.into(), + port: 443, + ..Default::default() + }; + let correlation = RequestCorrelation::new(&req, &ctx); + relay_http_request_with_response_middleware_guarded_observed( + &req, + &mut proxy_to_client, + &mut proxy_to_upstream, + RelayRequestOptions { + resolver: Some(&resolver), + credential_signing: signing, + signing_service: "s3", + signing_region: "us-east-1", + host, + port: 443, + ..Default::default() + }, + None, + None, + Some(&correlation), + ) + .await + }) + .with_subscriber(capture.subscriber()) + .await + .expect("relay must not deadlock"); + assert_eq!(result.is_ok(), succeeds, "{result:?}"); + drop(proxy_to_upstream); + let mut forwarded = String::new(); + upstream_side.read_to_string(&mut forwarded).await.unwrap(); + if succeeds { + assert!(forwarded.contains("authorization:"), "{forwarded}"); + } else { + assert!( + forwarded.is_empty(), + "signing failure must not reach upstream" + ); } - total += n; - if buf[..total].windows(4).any(|w| w == b"\r\n\r\n") { - upstream_side - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await - .unwrap(); - upstream_side.flush().await.unwrap(); - break; + let spans = capture.spans(); + let stages = spans + .iter() + .filter(|span| span.name == "openshell.credential.prepare") + .collect::>(); + assert_eq!(stages.len(), 2, "header resolution and signing"); + assert_eq!(stages[0].status, opentelemetry::trace::Status::Ok); + let signing = stages[1]; + assert_eq!( + matches!(signing.status, opentelemetry::trace::Status::Error { .. }), + !succeeds, + ); + assert_eq!( + span_attribute(signing, "openshell.request.failure_reason") + .map(|value| value.as_str()), + (!succeeds).then_some(Cow::Borrowed("credential_signing_failed")), + ); + if succeeds { + assert_eq!(signing.status, opentelemetry::trace::Status::Ok); } + capture.shutdown(); } - String::from_utf8_lossy(&buf[..total]).to_string() - }); - - let relay_result = tokio::time::timeout( - std::time::Duration::from_secs(5), - relay_http_request_with_options_guarded( - &req, - &mut proxy_to_client, - &mut proxy_to_upstream, - RelayRequestOptions { - resolver: Some(&resolver), - credential_signing: crate::l7::CredentialSigning::SigV4Body, - signing_service: "s3", - signing_region: "us-east-1", - host: "s3.us-east-1.amazonaws.com", - port: 443, - ..Default::default() - }, - ), - ) - .await - .expect("relay must not deadlock"); - drop(proxy_to_upstream); - let forwarded = upstream_task.await.expect("upstream task should complete"); - - relay_result.expect("bodyless GET with SigV4Body should succeed"); - assert!( - forwarded.contains("authorization:"), - "forwarded request should have SigV4 authorization header: {forwarded}" - ); + } } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index d189963a8b..85e3a602b1 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -6,6 +6,7 @@ mod http_hooks; use super::*; +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; /// Default wall-clock bound shared by whole-body stages in one response. pub const DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT: std::time::Duration = @@ -53,6 +54,7 @@ pub(super) struct RelayResponseOptions<'a> { pub(super) client_requested_upgrade: bool, pub(super) websocket: Option, pub(super) observer: Option<&'a EndpointObserver>, + pub(super) correlation: Option<&'a RequestCorrelation>, } impl Default for RelayResponseOptions<'_> { @@ -62,6 +64,7 @@ impl Default for RelayResponseOptions<'_> { client_requested_upgrade: true, websocket: None, observer: None, + correlation: None, } } } @@ -84,7 +87,7 @@ where // Forward interim responses unchanged, but retain the final response head // until response middleware preflight completes. let mut informational_header_bytes = 0; - let header_end = loop { + let mut header_end = loop { let header_end = buf .windows(4) .position(|w| w == b"\r\n\r\n") @@ -106,10 +109,12 @@ where Some(100 | 102..=199) ) { informational_header_bytes += header_end; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let sanitized = if options.correlation.is_some() { + strip_platform_request_id_from_response_head(&buf[..header_end]) + } else { + Cow::Borrowed(&buf[..header_end]) + }; + client.write_all(&sanitized).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; buf.drain(..header_end); continue; @@ -122,6 +127,7 @@ where if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } + record_upstream_response_error(options.correlation, "upstream_response_incomplete"); if !buf.is_empty() { client.write_all(&buf).await.into_diagnostic()?; } @@ -130,6 +136,16 @@ where buf.extend_from_slice(&tmp[..n]); }; + if options.correlation.is_some() + && let Cow::Owned(sanitized) = + strip_platform_request_id_from_response_head(&buf[..header_end]) + { + let overflow = buf.split_off(header_end); + buf = sanitized; + header_end = buf.len(); + buf.extend_from_slice(&overflow); + } + // Parse response framing let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); @@ -172,10 +188,8 @@ where options.websocket_extensions, options.websocket.as_ref(), )?; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let final_head = final_response_head(&buf[..header_end], options.correlation); + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; let overflow = buf[header_end..].to_vec(); debug!( @@ -183,6 +197,7 @@ where overflow_bytes = overflow.len(), "101 Switching Protocols — signaling protocol upgrade" ); + record_allowed_response(options.correlation); return Ok(RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -202,6 +217,7 @@ where body_length, server_wants_close || http_10_closes_by_default, event_stream, + options.correlation, )) .await? { @@ -212,13 +228,15 @@ where }; } + let final_head = final_response_head(&buf[..header_end], options.correlation); + // Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body if is_bodiless_response(request_method, status_code) { - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; - return finish_response(client, server_wants_close || http_10_closes_by_default).await; + client.write_all(&final_head).await.into_diagnostic()?; + let outcome = + finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + return Ok(outcome); } // No explicit framing (no Content-Length, no Transfer-Encoding). @@ -227,7 +245,7 @@ where if server_wants_close || http_10_closes_by_default || event_stream { // Server indicated it will close, or this is a streaming response // such as SSE where the body is intentionally delimited by EOF. - let before_end = &buf[..header_end - 2]; + let before_end = &final_head[..final_head.len() - 2]; client.write_all(before_end).await.into_diagnostic()?; if server_wants_close || http_10_closes_by_default { client @@ -249,25 +267,21 @@ where } client.flush().await.into_diagnostic()?; client.shutdown().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Consumed); } // No Connection: close — an HTTP/1.1 keep-alive server that omits // framing headers has an empty body. Forward headers and continue // the relay loop instead of blocking on relay_until_eof. debug!("BodyLength::None without Connection: close — treating body as empty"); - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Reusable); } // Hold buffered chunk framing until relay_chunked validates it. - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; let overflow = &buf[header_end..]; let overflow_len = (buf.len() - header_end) as u64; @@ -283,7 +297,14 @@ where } } BodyLength::Chunked => { - relay_chunked(upstream, client, overflow, None).await?; + relay_chunked_with_receipt_protection( + upstream, + client, + overflow, + None, + options.correlation.is_some(), + ) + .await?; } BodyLength::None => unreachable!(), } @@ -294,7 +315,166 @@ where "relay_response complete (explicit framing)" ); - finish_response(client, server_wants_close || http_10_closes_by_default).await + let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + Ok(outcome) +} + +fn record_allowed_response(correlation: Option<&RequestCorrelation>) { + if let Some(correlation) = correlation { + correlation.record_outcome(HttpOutcome::Allowed); + } +} + +fn record_denied_response(correlation: Option<&RequestCorrelation>, stage: &'static str) { + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, Some(stage)); + } +} + +fn record_upstream_response_error(correlation: Option<&RequestCorrelation>, stage: &'static str) { + if let Some(correlation) = correlation { + correlation.record_failure(HttpOutcome::UpstreamError, stage); + } +} + +fn record_committed_response_error( + correlation: Option<&RequestCorrelation>, + outcome: HttpOutcome, + stage: &'static str, +) { + if let Some(correlation) = correlation { + correlation.record_failure(outcome, stage); + } +} + +fn final_response_head<'a>( + headers: &'a [u8], + correlation: Option<&RequestCorrelation>, +) -> Cow<'a, [u8]> { + let Some(correlation) = correlation else { + return Cow::Borrowed(headers); + }; + if let Some(status_code) = parse_status_code(&String::from_utf8_lossy(headers)) { + correlation.record_response_status(status_code); + } + let stripped = strip_platform_request_id_from_response_head(headers); + let Some(header_end) = stripped.windows(4).position(|window| window == b"\r\n\r\n") else { + return stripped; + }; + let name = openshell_core::token_grant_failure::REQUEST_ID_HEADER; + let request_id = correlation.request_id(); + let mut output = Vec::with_capacity(stripped.len() + name.len() + request_id.len() + 4); + output.extend_from_slice(&stripped[..header_end + 2]); + output.extend_from_slice(name.as_bytes()); + output.extend_from_slice(b": "); + output.extend_from_slice(request_id.as_bytes()); + output.extend_from_slice(b"\r\n\r\n"); + output.extend_from_slice(&stripped[header_end + 4..]); + Cow::Owned(output) +} + +/// Remove only receipt fields and their hop-by-hop/trailer nominations. +/// Folded fields are grouped so removing a receipt never leaves its +/// continuation attached to an unrelated field. Recognize physical LF lines +/// without normalizing unrelated bytes or tightening response parsing. +fn strip_platform_request_id_from_response_head(headers: &[u8]) -> Cow<'_, [u8]> { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut output: Option> = None; + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(header_block.len(), |end| position + end); + let raw_line = &header_block[position..line_end]; + let line = raw_line.strip_suffix(b"\r").unwrap_or(raw_line); + if line.is_empty() { + if let Some(output) = &mut output { + output.extend_from_slice(&header_block[position..]); + } + break; + } + + let mut field_end = (line_end + 1).min(header_block.len()); + while header_block + .get(field_end) + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + field_end = header_block[field_end..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(header_block.len(), |end| field_end + end + 1); + } + let field = &header_block[position..field_end]; + if let Some(colon) = line.iter().position(|byte| *byte == b':') { + let name = trim_ascii_ows(&line[..colon]); + let receipt = name.eq_ignore_ascii_case(protected); + let nomination = (name.eq_ignore_ascii_case(b"connection") + || name.eq_ignore_ascii_case(b"trailer")) + && field + .windows(protected.len()) + .any(|window| window.eq_ignore_ascii_case(protected)); + let unfolded = nomination.then(|| { + let mut value = Vec::with_capacity(field.len() - colon); + for segment in field[colon + 1..].split(|byte| *byte == b'\n') { + let segment = trim_ascii_ows(segment.strip_suffix(b"\r").unwrap_or(segment)); + if !value.is_empty() && !segment.is_empty() { + value.push(b' '); + } + value.extend_from_slice(segment); + } + value + }); + let protected_nomination = unfolded.as_ref().is_some_and(|value| { + value + .split(|byte| *byte == b',') + .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)) + }); + if receipt || protected_nomination { + let output = output.get_or_insert_with(|| { + let mut output = Vec::with_capacity(headers.len()); + output.extend_from_slice(&header_block[..position]); + output + }); + if let Some(unfolded) = unfolded { + let mut retained = unfolded + .split(|byte| *byte == b',') + .map(trim_ascii_ows) + .filter(|token| { + !token.is_empty() && !token.eq_ignore_ascii_case(protected) + }); + if let Some(first) = retained.next() { + output.extend_from_slice(&line[..=colon]); + output.push(b' '); + output.extend_from_slice(first); + for token in retained { + output.extend_from_slice(b", "); + output.extend_from_slice(token); + } + output.extend_from_slice(b"\r\n"); + } + } + position = field_end; + continue; + } + } + if let Some(output) = &mut output { + output.extend_from_slice(field); + } + position = field_end; + } + + if let Some(output) = &mut output { + output.extend_from_slice(buffered_suffix); + } + output.map_or(Cow::Borrowed(headers), Cow::Owned) } /// Body framing determines when delivery finishes, not whether another @@ -314,6 +494,12 @@ where } } +/// Preserve the stage's failure when its local 502 was delivered successfully. +enum ResponseMiddlewareResult { + Relayed(Option), + Failed(&'static str), +} + #[allow(clippy::too_many_arguments)] async fn relay_response_through_middleware( request_method: &str, @@ -326,6 +512,7 @@ async fn relay_response_through_middleware( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result> where U: AsyncRead + Unpin, @@ -351,6 +538,64 @@ where } Err(error) => Err(error), }; + let response = relay_described_response_through_middleware( + request_method, + upstream, + client, + middleware, + buffered, + header_end, + status_code, + body_length, + server_wants_close, + event_stream, + response_correlation, + parsed, + ); + let response = if let Some(correlation) = response_correlation { + Box::pin(correlation.run_stage_classified( + RequestStage::ResponseMiddleware, + HttpOutcome::PlatformError, + "response_middleware_failed", + response, + |result| match result { + ResponseMiddlewareResult::Relayed(_) => None, + ResponseMiddlewareResult::Failed(reason) => Some(*reason), + }, + )) + .await + } else { + response.await + }?; + Ok(match response { + ResponseMiddlewareResult::Relayed(outcome) => outcome, + ResponseMiddlewareResult::Failed(_) => Some(RelayOutcome::Consumed), + }) +} + +#[allow(clippy::too_many_arguments)] +async fn relay_described_response_through_middleware( + request_method: &str, + upstream: &mut U, + client: &mut C, + middleware: HttpResponseMiddlewareRelay<'_>, + buffered: &[u8], + header_end: usize, + status_code: u16, + body_length: BodyLength, + server_wants_close: bool, + event_stream: bool, + response_correlation: Option<&RequestCorrelation>, + parsed: Result<( + Vec, + ParsedResponseHead, + )>, +) -> Result +where + U: AsyncRead + Unpin, + C: AsyncWrite + Unpin, +{ + let header_bytes = &buffered[..header_end]; if let Some(guard) = middleware.generation_guard { guard.ensure_current()?; } @@ -359,6 +604,7 @@ where Err(error) => { debug!(error = %error, "HTTP response head normalization failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -370,9 +616,14 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_headers", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_headers", + )); } }; match openshell_supervisor_middleware::chain_http_hook_version(&described) { @@ -391,6 +642,7 @@ where body_length, server_wants_close, event_stream, + response_correlation, )) .await; } @@ -400,6 +652,7 @@ where "HTTP response middleware chain mixes HTTP hook versions" ); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -411,9 +664,14 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_hook_version", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_hook_version", + )); } } let original_headers = parsed.headers.clone(); @@ -448,6 +706,7 @@ where Err(error) => { debug!(error = %error, "HTTP response middleware preflight failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -459,9 +718,14 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_preflight", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_preflight", + )); } }; if let Some(guard) = middleware.generation_guard @@ -499,7 +763,7 @@ where )); } } - crate::telemetry::emit_decision(preflight_events, None); + crate::l7::correlation::emit_request_events(response_correlation, preflight_events); if !preflight.allowed { if let Some(denial) = preflight.denial.as_ref() { send_response_middleware_denial( @@ -508,6 +772,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_preflight", + response_correlation, ) .await?; } else { @@ -517,24 +783,31 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_preflight", + response_correlation, ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if preflight.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_preflight") + }); } - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_decision_events( middleware.policy_name, &middleware.target, status_code, &preflight.invocations, ), - None, ); let Some(mut session) = preflight.session else { if preflight.headers == original_headers { - return Ok(None); + return Ok(ResponseMiddlewareResult::Relayed(None)); } let status_line = response_status_line(header_bytes)?; let outcome = relay_headers_only_response( @@ -550,9 +823,11 @@ where body_length, server_wants_close, event_stream, + response_correlation, ) .await?; - return Ok(Some(outcome)); + record_allowed_response(response_correlation); + return Ok(ResponseMiddlewareResult::Relayed(Some(outcome))); }; let status_line = response_status_line(header_bytes)?; @@ -577,23 +852,29 @@ where false, true, )); - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); send_response_delivery_failure( client, request_method, middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_finish", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_finish", + )); } }; if let Some(guard) = middleware.generation_guard { guard.ensure_current()?; } let mut headers = preflight.headers; - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_diagnostic_events( middleware.policy_name, &middleware.target, @@ -601,7 +882,6 @@ where &finish.invocations, &finish.findings, ), - None, ); if finish.strip_stale_integrity_headers { strip_response_integrity_headers(&mut headers); @@ -614,13 +894,17 @@ where server_wants_close, &[], ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; - return Ok(Some(if server_wants_close { - RelayOutcome::Consumed - } else { - RelayOutcome::Reusable - })); + record_allowed_response(response_correlation); + return Ok(ResponseMiddlewareResult::Relayed(Some( + if server_wants_close { + RelayOutcome::Consumed + } else { + RelayOutcome::Reusable + }, + ))); } let whole_body = session.requires_whole_body(); @@ -648,6 +932,7 @@ where server_wants_close || close_delimited_output, downstream_trailers, ); + let streaming_head = final_response_head(&streaming_head, response_correlation); let mut committed = !whole_body; if committed { if let Err(error) = client.write_all(&streaming_head).await { @@ -681,6 +966,7 @@ where &middleware.target, status_code, &connection_nominated_headers, + response_correlation, ) .await; let trailers = match body_result { @@ -724,8 +1010,21 @@ where middleware_stop.is_some(), )); } - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); if committed { + if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { + record_committed_response_error( + response_correlation, + HttpOutcome::Denied, + "response_middleware_body", + ); + } else { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "response_middleware_body", + ); + } return Err(error); } debug!(error = %error, "HTTP response processing failed before commitment"); @@ -736,6 +1035,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_body", + response_correlation, ) .await?; } else { @@ -745,10 +1046,19 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_body", + response_correlation, ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok( + if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_body") + }, + ); } }; @@ -780,8 +1090,21 @@ where true, )); } - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); if committed { + if error.denial.is_some() { + record_committed_response_error( + response_correlation, + HttpOutcome::Denied, + "response_middleware_finish", + ); + } else { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "response_middleware_finish", + ); + } return Err(miette!( "HTTP response middleware failed after commitment: {error}" )); @@ -794,6 +1117,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_finish", + response_correlation, ) .await?; } else { @@ -803,16 +1128,24 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_finish", + response_correlation, ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if error.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_finish") + }); } }; if let Some(guard) = middleware.generation_guard { guard.ensure_current()?; } - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_diagnostic_events( middleware.policy_name, &middleware.target, @@ -820,7 +1153,6 @@ where &finish.invocations, &finish.findings, ), - None, ); if whole_body && !committed { @@ -855,6 +1187,7 @@ where server_wants_close, &trailer_names, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if matches!(framing, ResponseFraming::Chunked) { for unit in &finish.body_units { @@ -879,7 +1212,8 @@ where } } client.flush().await.into_diagnostic()?; - Ok(Some( + record_allowed_response(response_correlation); + Ok(ResponseMiddlewareResult::Relayed(Some( if server_wants_close || (committed && close_delimited_output) || (matches!(body_length, BodyLength::None) && event_stream) @@ -888,7 +1222,7 @@ where } else { RelayOutcome::Reusable }, - )) + ))) } #[allow(clippy::too_many_arguments)] @@ -905,6 +1239,7 @@ async fn relay_headers_only_response( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result where U: AsyncRead + Unpin, @@ -918,6 +1253,7 @@ where server_wants_close, declared_trailers, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if is_bodiless_response(request_method, status_code) { @@ -939,7 +1275,16 @@ where relay_fixed(upstream, client, remaining, None).await?; } } - BodyLength::Chunked => relay_chunked(upstream, client, overflow, None).await?, + BodyLength::Chunked => { + relay_chunked_with_receipt_protection( + upstream, + client, + overflow, + None, + response_correlation.is_some(), + ) + .await?; + } BodyLength::None if server_wants_close || event_stream => { if !overflow.is_empty() { client.write_all(overflow).await.into_diagnostic()?; @@ -1002,12 +1347,14 @@ fn http_response_diagnostic_events( } fn emit_http_response_diagnostics( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, diagnostics: &openshell_supervisor_middleware::HttpResponseDiagnostics, ) { - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + correlation, http_response_diagnostic_events( policy_name, target, @@ -1015,7 +1362,6 @@ fn emit_http_response_diagnostics( &diagnostics.invocations, &diagnostics.findings, ), - None, ); } @@ -1035,7 +1381,7 @@ pub(super) fn http_response_middleware_invocation_events( openshell_supervisor_middleware::HttpResponseInvocationOutcome::BlockDelivery | openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailClosed ); - openshell_ocsf::HttpActivityBuilder::new(ocsf_ctx()) + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(openshell_ocsf::ActivityId::Other) .action(if blocked { openshell_ocsf::ActionId::Denied @@ -1076,6 +1422,8 @@ pub(super) fn http_response_middleware_invocation_events( u16::try_from(target.port).unwrap_or_default(), )) .firewall_rule(policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "response") .unmapped("middleware_config", invocation.config_name.as_str()) .unmapped( "middleware_implementation", @@ -1183,18 +1531,22 @@ pub(super) fn http_response_middleware_fail_open_finding_event( } fn emit_http_response_middleware_failure( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, committed: bool, ) { - crate::telemetry::egress_emit!(http_response_middleware_failure_event( - policy_name, - target, - status_code, - committed, - true, - )); + crate::l7::correlation::emit_request_events( + correlation, + [http_response_middleware_failure_event( + policy_name, + target, + status_code, + committed, + true, + )], + ); } fn http_response_middleware_failure_event( @@ -1361,7 +1713,8 @@ fn validate_http_field_value(value: &str) -> Result<()> { } fn is_protected_response_field(name: &str) -> bool { - name.eq_ignore_ascii_case("content-length") + name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + || name.eq_ignore_ascii_case("content-length") || is_hidden_response_field(name) || openshell_supervisor_middleware::headers::is_response_credential_header(name) } @@ -1544,6 +1897,7 @@ async fn relay_normalized_response_body( target: &HttpRequestTarget, status_code: u16, connection_nominated_headers: &[String], + correlation: Option<&RequestCorrelation>, ) -> Result> where R: AsyncRead + Unpin, @@ -1558,6 +1912,7 @@ where policy_name, target, status_code, + correlation, }; match body_length { BodyLength::ContentLength(mut remaining) => { @@ -1761,6 +2116,7 @@ struct ResponseOutputState<'a> { policy_name: &'a str, target: &'a HttpRequestTarget, status_code: u16, + correlation: Option<&'a RequestCorrelation>, } #[derive(Debug)] @@ -1824,6 +2180,7 @@ async fn expire_whole_body_deadline( let diagnostics = session.take_diagnostics(); if output.is_ok() { emit_http_response_diagnostics( + framing.correlation, framing.policy_name, framing.target, framing.status_code, @@ -1968,6 +2325,7 @@ async fn process_response_unit( let diagnostics = session.take_diagnostics(); if output.is_ok() { emit_http_response_diagnostics( + framing.correlation, framing.policy_name, framing.target, framing.status_code, @@ -2050,12 +2408,17 @@ where C: AsyncWrite + Unpin, { let mut trailers = Vec::new(); + let mut field_count = 0; loop { let line = read_response_line_with_deadline(reader, session, client, framing).await?; validate_chunk_trailer_line(&line)?; if line.is_empty() { return Ok(trailers); } + field_count += 1; + if field_count > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { + return Err(miette!("HTTP response trailer count exceeds limit")); + } let line = std::str::from_utf8(&line) .map_err(|_| miette!("HTTP response trailer contains invalid UTF-8"))?; let (name, value) = line @@ -2064,6 +2427,11 @@ where validate_http_field_name(name)?; validate_http_field_value(value.trim())?; let name = name.to_ascii_lowercase(); + // The upstream cannot supply the platform receipt. Strip it just as + // the ordinary response relay does, before exposing trailers to stages. + if name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) { + continue; + } if is_protected_response_field(&name) || connection_nominated_headers.contains(&name) { return Err(miette!("HTTP response trailer uses a protected field name")); } @@ -2071,9 +2439,6 @@ where name, value: value.trim().to_string(), }); - if trailers.len() > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { - return Err(miette!("HTTP response trailer count exceeds limit")); - } } } @@ -2098,7 +2463,10 @@ async fn send_response_middleware_denial( policy_name: &str, target: &HttpRequestTarget, denial: &openshell_supervisor_middleware::MiddlewareDenial, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + record_denied_response(correlation, stage); let mut body = serde_json::Map::new(); body.insert("error".into(), serde_json::json!("middleware_denied")); body.insert( @@ -2123,7 +2491,8 @@ async fn send_response_middleware_denial( "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2133,13 +2502,25 @@ async fn send_response_middleware_denial( /// Answer a response that middleware failed before its head was committed. /// An HTTP hook failure carries its platform-owned code and stage. +#[allow(clippy::too_many_arguments)] async fn send_response_delivery_failure( client: &mut C, request_method: &str, policy_name: &str, target: &HttpRequestTarget, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + outcome: HttpOutcome, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + if let Some(correlation) = correlation { + if outcome == HttpOutcome::UpstreamError { + correlation.record_response_status(502); + correlation.record_failure(outcome, stage); + } else { + correlation.record_rejection(502, outcome, Some(stage)); + } + } let mut body = serde_json::json!({ "error": "response_delivery_failed", "detail": "The upstream request may have completed, but OpenShell could not deliver its response. Retrying may repeat upstream side effects.", @@ -2158,7 +2539,8 @@ async fn send_response_delivery_failure( "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2237,9 +2619,13 @@ pub(super) fn response_is_event_stream(headers: &str) -> bool { #[cfg(test)] mod tests { + use super::{final_response_head, strip_platform_request_id_from_response_head}; + use crate::l7::correlation::HttpOutcome; + use crate::l7::rest::strip_platform_request_id_from_head; use openshell_supervisor_middleware::{ HttpResponseInvocation, HttpResponseInvocationOutcome as Outcome, NamespacedFinding, }; + use std::borrow::Cow; fn invocation(outcome: Outcome, failed: bool) -> HttpResponseInvocation { HttpResponseInvocation { @@ -2278,8 +2664,9 @@ mod tests { for committed in [false, true] { let (events, received) = exported("warn", || { super::emit_http_response_middleware_failure( + None, "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, committed, ); @@ -2312,8 +2699,9 @@ mod tests { ..Default::default() }; super::emit_http_response_diagnostics( + None, "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, &diagnostics, ); @@ -2340,7 +2728,7 @@ mod tests { #[test] fn fail_closed_invocations_and_enforced_failures_record_the_denial() { - let target = super::HttpRequestTarget::default(); + let target = HttpRequestTarget::default(); for (outcome, failed, action_id) in [ (Outcome::BlockDelivery, false, 2), (Outcome::FailClosed, true, 2), @@ -2368,12 +2756,668 @@ mod tests { } } + #[test] + fn late_fail_open_invocation_does_not_hide_block_delivery() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("late-response-receipt"); + let target = HttpRequestTarget { + method: "GET".into(), + scheme: "https".into(), + host: "example.test".into(), + path: "/data".into(), + port: 443, + ..Default::default() + }; + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "GET", + openshell_ocsf::Url::new("https", "example.test", "/data", 443), + )) + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + correlation.record_response_status(200); + for (outcome, failed) in [(Outcome::FailOpen, true), (Outcome::BlockDelivery, false)] { + for event in super::http_response_middleware_invocation_events( + "response-policy", + &target, + 200, + &[invocation(outcome, failed)], + ) { + correlation.emit(event); + } + } + drop(correlation); + let events: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(events.len(), 3); + assert_eq!(events[0]["unmapped"]["http_event_kind"], "decision"); + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[1]["action"], "Other"); + assert_eq!( + events[1]["unmapped"]["response_middleware_outcome"], + "failopen" + ); + assert_eq!(events[2]["action"], "Denied"); + for event in &events[1..] { + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + assert_eq!(event["http_response"]["code"], 200); + } + assert_eq!( + events[2]["unmapped"]["response_middleware_outcome"], + "blockdelivery" + ); + assert!( + events + .iter() + .all(|event| event["metadata"]["correlation_uid"] == "late-response-receipt") + ); + } + use super::{ - ResponseFraming, parse_connection_keep_alive, parse_response_head_for_middleware, - response_is_http_10, serialize_response_head, + RelayResponseOptions, ResponseFraming, parse_connection_keep_alive, + parse_response_head_for_middleware, relay_response, response_is_http_10, + serialize_response_head, + }; + use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; + use crate::l7::relay::L7EvalContext; + use crate::l7::rest::tests::{ + ResponseRelayScript, response_middleware_context, response_middleware_fixture, }; - use crate::l7::provider::BodyLength; - use openshell_core::proto::HttpHeader; + use openshell_core::proto::{HttpHeader, HttpRequestTarget}; + use std::collections::HashMap; + + async fn relay_bytes( + mut upstream: &[u8], + correlation: Option<&crate::l7::correlation::RequestCorrelation>, + ) -> (miette::Result, Vec) { + let mut client = Vec::new(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation, + ..Default::default() + }, + None, + ) + .await; + (outcome, client) + } + + fn test_correlation_for_method( + request_id: &str, + method: &str, + ) -> crate::l7::correlation::RequestCorrelation { + let request = L7Request { + action: method.into(), + target: "/data".into(), + query_params: HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + request_id.into(), + ) + } + + fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { + test_correlation_for_method(request_id, "GET") + } + + #[test] + fn untouched_receipt_heads_remain_borrowed() { + let head = b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nConnection: keep-alive\r\nTrailer: X-Checksum\r\n\r\nbody"; + let uncorrelated = final_response_head(head, None); + assert!(matches!(uncorrelated, Cow::Borrowed(_))); + assert_eq!(uncorrelated.as_ref(), head); + + let stripped = strip_platform_request_id_from_head(head); + assert!(matches!(stripped, Cow::Borrowed(_))); + let stripped = strip_platform_request_id_from_response_head(&stripped); + assert!(matches!(stripped, Cow::Borrowed(_))); + assert_eq!(stripped.as_ref(), head); + } + + #[test] + fn receipt_head_rewrite_preserves_unrelated_bytes_and_buffered_body() { + let correlation = test_correlation("receipt-cow"); + let head = b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nX-OpenShell-Request-ID: forged\r\nx-openshell-request-id: forged-again\r\nConnection: keep-alive, X-OpenShell-Request-ID, , Upgrade\r\nTrailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n\x00\xffbody"; + + let rewritten = final_response_head(head, Some(&correlation)); + assert!(matches!(rewritten, Cow::Owned(_))); + assert_eq!( + rewritten.as_ref(), + b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nConnection: keep-alive, Upgrade\r\nTrailer: X-Checksum\r\nx-openshell-request-id: receipt-cow\r\n\r\n\x00\xffbody" + ); + } + + #[test] + fn clean_correlated_head_adds_only_the_receipt() { + let correlation = test_correlation("receipt-clean"); + let head = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"; + + let rewritten = final_response_head(head, Some(&correlation)); + assert_eq!( + rewritten.as_ref(), + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nx-openshell-request-id: receipt-clean\r\n\r\nbody" + ); + } + + fn request_id_header_count(response: &[u8]) -> usize { + String::from_utf8_lossy(response) + .lines() + .filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case( + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + }) + }) + .count() + } + + #[tokio::test] + async fn allowed_final_response_carries_platform_receipt() { + let correlation = test_correlation("receipt-1"); + let (outcome, client) = relay_bytes( + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody", + Some(&correlation), + ) + .await; + + assert!(matches!( + outcome.expect("relay response"), + RelayOutcome::Reusable + )); + assert_eq!(request_id_header_count(&client), 1); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn delivery_failure_receipt_does_not_synthesize_audit_records() { + for outcome in [HttpOutcome::UpstreamError, HttpOutcome::PlatformError] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("delivery-failure"); + let target = HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }; + let mut client = Vec::new(); + super::send_response_delivery_failure( + &mut client, + "GET", + "policy", + &target, + None, + outcome, + "response_body", + Some(&correlation), + ) + .await + .unwrap(); + drop(correlation); + assert_eq!(request_id_header_count(&client), 1); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + assert!(capture.json_events().is_empty()); + } + } + + #[tokio::test] + async fn correlated_upstream_eof_preserves_transport_close_without_synthetic_records() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("eof-request"); + let (outcome, client) = relay_bytes(b"", Some(&correlation)).await; + assert!(matches!( + outcome.expect("upstream close"), + RelayOutcome::Consumed + )); + assert!( + client.is_empty(), + "EOF must not become a synthetic HTTP response" + ); + drop(correlation); + assert!(capture.json_events().is_empty()); + } + + #[tokio::test] + async fn correlated_response_preserves_legacy_status_line_tolerance() { + for status in [ + "HTTP/1.1 200", + "HTTP/1.1 200 OK", + "HTTP/1.1\t200 OK", + "HTTP/1.2 200 OK", + ] { + let response = format!("{status}\r\nContent-Length: 0\r\n\r\n"); + let (outcome, client) = relay_bytes(response.as_bytes(), None).await; + outcome.expect("legacy status line"); + assert_eq!(client, response.as_bytes()); + + let correlation = test_correlation("legacy-status"); + let (outcome, client) = relay_bytes(response.as_bytes(), Some(&correlation)).await; + outcome.unwrap_or_else(|error| panic!("{status:?}: {error}")); + assert!(client.starts_with(format!("{status}\r\n").as_bytes())); + assert_eq!(request_id_header_count(&client), 1); + assert_eq!( + crate::test_support::response_request_id(&client), + "legacy-status" + ); + } + } + + #[tokio::test] + async fn uncorrelated_response_preserves_application_owned_request_id() { + let response = + b"HTTP/1.1 200\r\nContent-Length: 0\r\nX-OpenShell-Request-ID: application\r\n\r\n"; + let (outcome, client) = relay_bytes(response, None).await; + outcome.expect("legacy response"); + assert_eq!(client, response); + } + + #[tokio::test] + async fn response_middleware_denial_preserves_correlated_control_records_without_tracing() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + exercise_response_middleware_block().await; + + assert_response_middleware_block_record(&capture.json_events(), false); + } + + async fn exercise_response_middleware_block() { + let correlation = test_correlation_for_method("response-denied", "POST"); + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "POST", + openshell_ocsf::Url::new("https", "example.test", "/mutate", 443), + )) + .firewall_rule("test-policy", "l7-rest") + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::BlockPreflight); + let mut middleware = response_middleware_context(&runner, &chain, "POST"); + middleware.request_context.request_id = "response-denied".into(); + middleware.target.path = "/mutate".into(); + middleware.request_head = None; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "POST", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("middleware denial response"); + + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); + assert!( + String::from_utf8_lossy(&client) + .contains("x-openshell-request-id: response-denied\r\n") + ); + drop(correlation); + } + + fn assert_response_middleware_block_record(records: &[serde_json::Value], traced: bool) { + let http: Vec<_> = records + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(http.len(), 2); + let companions: Vec<_> = http + .iter() + .filter(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation") + .collect(); + assert_eq!(companions.len(), 1); + assert_eq!(companions[0]["http_response"]["code"], 200); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "response"); + assert_eq!( + companions[0]["unmapped"]["response_middleware_outcome"], + "blockdelivery" + ); + assert_eq!(companions[0]["action"], "Denied"); + assert_eq!( + companions[0]["metadata"]["correlation_uid"], + "response-denied" + ); + let policy: Vec<_> = records + .iter() + .filter(|event| { + event["class_uid"] == 4002 && event["firewall_rule"]["type"] == "l7-rest" + }) + .collect(); + assert_eq!(policy.len(), 1, "{records:?}"); + assert_eq!(policy[0]["action"], "Allowed"); + assert_eq!(policy[0]["status"], "Success"); + assert!(policy[0].get("http_response").is_none()); + assert_eq!(policy[0]["http_request"]["url"]["path"], "/mutate"); + for record in records { + assert_eq!(record["metadata"]["correlation_uid"], "response-denied"); + if record["class_uid"] == 4002 { + assert_eq!(record.get("trace").is_some(), traced); + } + } + } + + #[tokio::test] + async fn response_middleware_block_never_creates_a_denial_only_span() { + use crate::l7::relay::correlation_dispatch_tests::capture_request_spans; + use crate::test_support::request_tracing::span_attribute; + + for sampled in [true, false] { + let evidence = Box::pin(capture_request_spans( + sampled, + exercise_response_middleware_block(), + )) + .await; + evidence.assert_no_deny_span(sampled); + assert_response_middleware_block_record(&evidence.events, sampled); + if sampled { + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.response") + .collect(); + assert_eq!(middleware.len(), 1); + assert_eq!(middleware[0].status, opentelemetry::trace::Status::Ok); + assert!( + span_attribute(middleware[0], "openshell.request.failure_reason").is_none() + ); + } + } + } + + #[tokio::test] + async fn response_middleware_operational_failure_marks_child_error_after_delivering_502() { + use crate::l7::correlation::RequestStage; + use crate::l7::relay::correlation_dispatch_tests::capture_request_spans; + use crate::test_support::request_tracing::span_attribute; + + let evidence = Box::pin(capture_request_spans(true, async { + let correlation = test_correlation("response-failed"); + let (runner, chain) = + response_middleware_fixture(ResponseRelayScript::InvalidWholeBodySequence); + let middleware = response_middleware_context(&runner, &chain, "GET"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + let outcome = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::PlatformError, + "response_relay_failed", + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ), + )) + .await + .expect("local failure response is delivered"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + })) + .await; + + evidence.assert_no_deny_span(true); + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.response") + .collect(); + assert_eq!(middleware.len(), 1); + assert!(matches!( + middleware[0].status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(middleware[0], "openshell.request.failure_reason") + .unwrap() + .as_str(), + "response_middleware_finish" + ); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .expect("response delivery stage"); + assert_eq!(response.status, opentelemetry::trace::Status::Ok); + assert!(span_attribute(response, "openshell.request.failure_reason").is_none()); + } + + #[tokio::test] + async fn upstream_receipt_is_replaced_by_platform_receipt() { + let correlation = test_correlation("trusted"); + let response = + &b"HTTP/1.1 200 OK\r\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("relay response"); + + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: trusted\r\n")); + assert_eq!(request_id_header_count(&client), 1); + } + + #[tokio::test] + async fn ordinary_response_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let response = &b"HTTP/1.1 200 OK\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id\r\n\ + X-OpenShell-Request-ID: forged-head\r\n\ + Trailer: x-openshell-request-id, X-Checksum\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n\ + 0\r\n\ + x-OPENshell-request-ID: forged-trailer\r\n\ + X-Checksum: ok\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let head = delivered + .split_once("\r\n\r\n") + .expect("response head terminator") + .0; + let connection_headers: Vec<_> = head + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!(head.contains("Trailer: X-Checksum"), "{head}"); + assert!(delivered.contains("0\r\nX-Checksum: ok\r\n\r\n")); + assert!(head.contains("x-openshell-request-id: trusted"), "{head}"); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn ordinary_response_sanitizes_only_folded_receipt_fields_and_nominations() { + let correlation = test_correlation("trusted"); + let response = &b"HTTP/1.1 200 OK\r\nX-Unrelated: opaque\r\n continuation\r\nX-OpenShell-Request-ID: forged\r\n more-forgery\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nTrailer: X-Checksum,\r\n\tx-openshell-request-id\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("protected folded fields are sanitized"); + assert_eq!( + client, + b"HTTP/1.1 200 OK\r\nX-Unrelated: opaque\r\n continuation\r\nConnection: keep-alive\r\nTrailer: X-Checksum\r\nContent-Length: 0\r\nx-openshell-request-id: trusted\r\n\r\n" + ); + } + + #[tokio::test] + async fn ordinary_response_strips_bare_lf_receipt_without_normalizing_unrelated_fields() { + let correlation = test_correlation("trusted"); + let response = b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nX-Unrelated: opaque\n retained\r\nContent-Length: 0\r\n\r\n"; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("protected physical LF fields are sanitized"); + + assert_eq!( + client, + b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-Unrelated: opaque\n retained\r\nContent-Length: 0\r\nx-openshell-request-id: trusted\r\n\r\n" + ); + assert_eq!(request_id_header_count(&client), 1); + } + + #[tokio::test] + async fn response_middleware_strips_upstream_receipt_trailers_without_aborting_streams() { + let correlation = test_correlation("trusted"); + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::Stream); + let mut middleware = response_middleware_context(&runner, &chain, "GET"); + middleware.request_context.request_id = "middleware-request".into(); + middleware.request_head = None; + let response = b"HTTP/1.1 200 OK\r\n\ + Transfer-Encoding: chunked\r\n\ + Trailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n\ + 4\r\nbody\r\n0\r\n\ + x-OPENshell-request-ID: forged\r\n\ + X-Checksum: ok\r\n\r\n"; + let (ordinary_outcome, ordinary) = relay_bytes(response, Some(&correlation)).await; + assert!(matches!(ordinary_outcome.unwrap(), RelayOutcome::Reusable)); + let mut upstream = response.as_slice(); + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("upstream receipt trailers are stripped without aborting delivery"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + let delivered = String::from_utf8_lossy(&client); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!(delivered.contains("4\r\nBODY\r\n"), "{delivered}"); + assert!( + delivered.ends_with("0\r\nx-checksum: ok\r\n\r\n"), + "{delivered}" + ); + assert!( + delivered.contains("x-openshell-request-id: trusted\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(&client), 1); + assert_eq!(request_id_header_count(&ordinary), 1); + } + + #[test] + fn response_middleware_rejects_declared_request_id_trailer() { + let error = parse_response_head_for_middleware( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-OpenShell-Request-ID\r\n\r\n", + ) + .expect_err("platform correlation trailer declarations must fail closed"); + + assert!( + error.to_string().contains("protected trailer field"), + "{error}" + ); + } + + #[tokio::test] + async fn informational_responses_are_not_decorated_but_upgrade_receives_a_receipt() { + let correlation = test_correlation("receipt-1"); + let response = + &b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("relay response"); + assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n")); + assert_eq!(request_id_header_count(&client), 1); + + let response = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + assert_eq!(request_id_header_count(&client), 1); + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn switching_protocols_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let response = &b"HTTP/1.1 101 Switching Protocols\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id, UpGrAdE\r\n\ + Upgrade: websocket\r\n\ + X-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let connection_headers: Vec<_> = delivered + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive, UpGrAdE"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted-upgrade\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn switching_protocols_sanitizes_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let response = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + assert!(matches!( + outcome.expect("sanitized upgrade"), + RelayOutcome::Upgraded { .. } + )); + assert!( + String::from_utf8_lossy(&client).contains("Connection: Upgrade\r\n"), + "{client:?}" + ); + assert_eq!(request_id_header_count(&client), 1); + } #[test] fn http_10_connection_persistence() { diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs index 6928d46b11..3e638c042a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs @@ -76,6 +76,15 @@ impl BodyRelayError { } } + const fn correlation_failure(&self) -> (HttpOutcome, &'static str) { + match self { + Self::Upstream(_) => (HttpOutcome::UpstreamError, "upstream_response_body"), + Self::Downstream(_) => (HttpOutcome::PlatformError, "downstream_write"), + Self::PolicyReload(_) => (HttpOutcome::PlatformError, "policy_generation_changed"), + Self::Output(_) => (HttpOutcome::PlatformError, "middleware_response_output"), + } + } + fn into_report(self) -> miette::Report { match self { Self::Upstream(error) | Self::PolicyReload(error) | Self::Output(error) => error, @@ -99,7 +108,8 @@ pub(super) async fn relay_response_through_pipeline( body_length: BodyLength, server_wants_close: bool, event_stream: bool, -) -> Result> + response_correlation: Option<&RequestCorrelation>, +) -> Result where U: AsyncRead + Unpin, C: AsyncWrite + Unpin, @@ -152,6 +162,7 @@ where Err(error) => { debug!(error = %error, "HTTP response middleware preflight failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -163,9 +174,14 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "middleware_response_preflight", + )); } }; if let Some(guard) = middleware.generation_guard @@ -174,9 +190,15 @@ where if let Some(session) = preflight.pipeline.take() { session.end(MiddlewareSessionEndReason::PolicyReload).await; } + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "policy_generation_changed", + ); return Err(error); } emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -191,10 +213,13 @@ where middleware.policy_name, &middleware.target, denial, + "middleware_response_preflight", + response_correlation, ) .await?; } else { emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -206,15 +231,22 @@ where middleware.policy_name, &middleware.target, preflight.failure.as_ref(), + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if preflight.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("middleware_response_preflight") + }); } let Some(session) = preflight.pipeline.take() else { if preflight.headers == original_headers { - return Ok(None); + return Ok(ResponseMiddlewareResult::Relayed(None)); } let outcome = relay_headers_only_response( request_method, @@ -229,9 +261,11 @@ where body_length, server_wants_close, event_stream, + response_correlation, ) .await?; - return Ok(Some(outcome)); + record_allowed_response(response_correlation); + return Ok(ResponseMiddlewareResult::Relayed(Some(outcome))); }; // Eligibility offers a bodyless response no body mode, so no stage can // have selected one. @@ -240,6 +274,7 @@ where .end(MiddlewareSessionEndReason::MiddlewareFailure) .await; emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -251,9 +286,14 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "middleware_response_preflight", + )); } let unit_limit = session.input_unit_limit(); @@ -276,6 +316,7 @@ where chunked, connection_close: server_wants_close, generation_guard: middleware.generation_guard, + correlation: response_correlation, framing: None, committed: false, withheld: Vec::new(), @@ -313,15 +354,23 @@ where let failure = match (finish, relayed) { (Ok(finish), Ok(())) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, &finish.diagnostics, ); if let Some(guard) = middleware.generation_guard { - guard.ensure_current()?; + guard.ensure_current().inspect_err(|_| { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "policy_generation_changed", + ); + })?; } - return Ok(Some( + record_allowed_response(response_correlation); + return Ok(ResponseMiddlewareResult::Relayed(Some( if server_wants_close || framing == Some(OutputFraming::CloseDelimited) || (matches!(body_length, BodyLength::None) && event_stream) @@ -330,10 +379,11 @@ where } else { RelayOutcome::Reusable }, - )); + ))); } (Err(failure), relayed) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -347,6 +397,7 @@ where } (Ok(finish), Err(error)) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -357,9 +408,14 @@ where }; let committed = writer.committed; match failure { - Err(error @ BodyRelayError::Downstream(_)) => Err(error.into_report()), + Err(error @ BodyRelayError::Downstream(_)) => { + record_relay_error(response_correlation, &error); + Err(error.into_report()) + } Err(error) if committed => { + record_relay_error(response_correlation, &error); emit_http_response_middleware_abort( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -369,8 +425,10 @@ where Err(error.into_report()) } Err(error) => { + let (outcome, stage) = error.correlation_failure(); debug!(error = %error.into_report(), "HTTP response relay failed before commitment"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -382,12 +440,26 @@ where middleware.policy_name, &middleware.target, None, + outcome, + stage, + response_correlation, ) .await?; - Ok(Some(RelayOutcome::Consumed)) + Ok(ResponseMiddlewareResult::Failed(stage)) } Ok(failure) if committed => { + if let Some(correlation) = response_correlation { + correlation.record_failure( + if failure.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + "middleware_response_finalize", + ); + } emit_http_response_middleware_abort( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -408,10 +480,13 @@ where middleware.policy_name, &middleware.target, denial, + "middleware_response_finalize", + response_correlation, ) .await?; } else { emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -423,14 +498,26 @@ where middleware.policy_name, &middleware.target, failure.failure.as_deref(), + HttpOutcome::PlatformError, + "middleware_response_finalize", + response_correlation, ) .await?; } - Ok(Some(RelayOutcome::Consumed)) + Ok(if failure.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("middleware_response_finalize") + }) } } } +fn record_relay_error(correlation: Option<&RequestCorrelation>, error: &BodyRelayError) { + let (outcome, reason) = error.correlation_failure(); + record_committed_response_error(correlation, outcome, reason); +} + /// Feed the upstream body to the stages in units of at most `limit` bytes. /// Upstream silence is never a failure: a stream ends when its framing ends. #[allow(clippy::too_many_arguments)] @@ -556,8 +643,15 @@ async fn read_trailer_fields( connection_nominated: &[String], ) -> Result> { let mut trailers = Vec::new(); + let mut field_count = 0; loop { let line = reader.read_line().await?; + if !line.is_empty() { + field_count += 1; + if field_count > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { + return Err(miette!("HTTP response trailer count exceeds limit")); + } + } if !push_trailer(&line, connection_nominated, &mut trailers)? { return Ok(trailers); } @@ -575,6 +669,7 @@ struct ResponseWriter<'c, 'a, C> { chunked: bool, connection_close: bool, generation_guard: Option<&'a PolicyGenerationGuard>, + correlation: Option<&'a RequestCorrelation>, /// Set on the output `Start`. framing: Option, /// True once any byte reached the client: no canonical error response may @@ -659,6 +754,11 @@ impl ResponseWriter<'_, '_, C> { &[] }, ); + let head = if self.correlation.is_some() { + final_response_head(&head, self.correlation).into_owned() + } else { + head + }; self.framing = Some(framing); // The head alone completes an empty declared body. if framing == (OutputFraming::ContentLength { remaining: 0 }) { @@ -730,14 +830,16 @@ impl ResponseWriter<'_, '_, C> { /// OCSF events for one phase of a response through the stage pipeline. fn emit_http_response_stage_events( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, diagnostics: &HttpStageDiagnostics, ) { - for event in http_response_stage_events(policy_name, target, status_code, diagnostics) { - crate::telemetry::egress_emit!(event); - } + crate::l7::correlation::emit_request_events( + correlation, + http_response_stage_events(policy_name, target, status_code, diagnostics), + ); } fn http_response_stage_events( @@ -831,6 +933,8 @@ fn http_response_stage_event( .http_response(openshell_ocsf::HttpResponse { code: status_code }) .dst_endpoint(openshell_ocsf::Endpoint::from_domain(&target.host, port)) .firewall_rule(policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "response") .unmapped("middleware_config", invocation.config_name.as_str()) .unmapped( "middleware_implementation", @@ -865,6 +969,11 @@ fn push_trailer( validate_http_field_name(name)?; validate_http_field_value(value.trim())?; let name = name.to_ascii_lowercase(); + // Only the platform owns receipt fields. Do not abort a valid upstream + // stream or expose its forged receipt to a response middleware stage. + if name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) { + return Ok(true); + } if is_protected_response_field(&name) || connection_nominated.contains(&name) { return Err(miette!("HTTP response trailer uses a protected field name")); } @@ -882,6 +991,7 @@ fn push_trailer( /// upstream failed, so delivery was aborted. `denied` is true when a stage /// stopped delivery, matching the legacy delivery failure finding. fn emit_http_response_middleware_abort( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, @@ -908,12 +1018,215 @@ fn emit_http_response_middleware_abort( ]) .message("HTTP response middleware failed after response commitment; delivery aborted") .build(); - crate::telemetry::egress_emit!(event); + crate::l7::correlation::emit_request_events(correlation, [event]); } #[cfg(test)] mod tests { use super::*; + use crate::test_support::request_tracing::RequestTracing; + use opentelemetry_sdk::trace::Sampler; + use tokio::io::AsyncReadExt as _; + use tracing::instrument::WithSubscriber as _; + + #[test] + fn upstream_receipt_trailers_are_hidden_from_http_hooks() { + let mut trailers = Vec::new(); + let nominated = vec!["x-openshell-request-id".into(), "x-private".into()]; + assert!( + push_trailer(b"x-OPENshell-request-ID: forged", &nominated, &mut trailers).unwrap() + ); + assert!(trailers.is_empty()); + assert!(push_trailer(b"X-Checksum: ok", &nominated, &mut trailers).unwrap()); + assert_eq!(trailers.len(), 1); + assert_eq!(trailers[0].name, "x-checksum"); + assert_eq!(trailers[0].value, "ok"); + assert!(!push_trailer(b"", &nominated, &mut trailers).unwrap()); + + for line in [ + b"Content-Length: 7".as_slice(), + b"Set-Cookie: secret", + b"X-Private: opaque", + b"X-OpenShell-Request-ID: bad\0value", + ] { + push_trailer(line, &nominated, &mut Vec::new()) + .expect_err("other protected or malformed trailers still fail closed"); + } + } + + #[test] + fn response_hook_invocations_keep_individual_correlated_records_without_tracing() { + let capture = RequestTracing::new(None); + tracing::subscriber::with_default(capture.subscriber(), || { + let request = L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: HashMap::default(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let correlation = RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + "response-audit".into(), + ); + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(ocsf_ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "GET", + openshell_ocsf::Url::new("https", "example.test", "/data", 443), + )) + .firewall_rule("request-policy", "l7-rest") + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + for (outcome, input_bytes) in [ + (HttpStageOutcome::WholeBody, 0), + (HttpStageOutcome::Unchanged, 5), + ] { + emit_http_response_stage_events( + Some(&correlation), + "response-policy", + &HttpRequestTarget::default(), + 200, + &HttpStageDiagnostics { + invocations: vec![HttpStageInvocation { + config_name: "guard".into(), + implementation: "test/guard".into(), + outcome, + input_bytes, + output_bytes: None, + transformed: false, + failed: false, + reason_code: None, + failure_reason: None, + }], + ..Default::default() + }, + ); + if outcome == HttpStageOutcome::WholeBody { + correlation.record_response_status(200); + } + } + }); + + let records = capture.events.json_events(); + assert_eq!(records.len(), 3); + let policy = &records[0]; + assert_eq!(policy["firewall_rule"]["type"], "l7-rest"); + assert_eq!(policy["action"], "Allowed"); + assert!(policy.get("http_response").is_none()); + let invocations = &records[1..]; + for (event, (outcome, input_bytes)) in + invocations.iter().zip([("wholebody", 0), ("unchanged", 5)]) + { + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + assert_eq!(event["unmapped"]["middleware_config"], "guard"); + assert_eq!(event["unmapped"]["middleware_implementation"], "test/guard"); + assert_eq!(event["unmapped"]["http_hook_version"], "2"); + assert_eq!(event["unmapped"]["response_middleware_outcome"], outcome); + assert_eq!(event["unmapped"]["input_bytes"], input_bytes); + assert_eq!(event["unmapped"]["failed"], false); + assert_eq!(event["http_response"]["code"], 200); + assert_eq!(event["action"], "Allowed"); + } + for event in &records { + assert_eq!(event["metadata"]["correlation_uid"], "response-audit"); + assert!(event.get("trace").is_none()); + } + capture.shutdown(); + } + + #[tokio::test] + async fn empty_pipeline_response_defers_its_protected_receipt_until_completion() { + for sampler in [None, Some(Sampler::AlwaysOff), Some(Sampler::AlwaysOn)] { + let capture = RequestTracing::new(sampler); + async { + let request = L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: HashMap::default(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let correlation = RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + "pipeline-receipt".into(), + ); + let parsed = ParsedResponseHead { + representable: true, + headers: vec![HttpHeader { + name: "x-openshell-request-id".into(), + value: "upstream-forgery".into(), + }], + preserved_credential_headers: Vec::new(), + connection_nominated: Vec::new(), + declared_trailers: Vec::new(), + }; + let (mut relay, mut app) = tokio::io::duplex(4096); + { + let mut writer = ResponseWriter { + client: &mut relay, + status_line: "HTTP/1.1 200 OK", + headers: &parsed.headers, + parsed: &parsed, + chunked: true, + connection_close: false, + generation_guard: None, + correlation: Some(&correlation), + framing: None, + committed: false, + withheld: Vec::new(), + }; + writer + .write(HttpBodyOutput::Start { + header_mutations: Vec::new(), + output_body_bytes: Some(0), + }) + .await + .unwrap(); + assert!(!writer.committed); + assert!(capture.events.json_events().is_empty()); + writer + .write(HttpBodyOutput::End { + trailers: Vec::new(), + }) + .await + .unwrap(); + assert!(writer.committed); + } + drop(relay); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + let response = std::str::from_utf8(&response).unwrap(); + let mut receipts = response.lines().filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case("x-openshell-request-id") + }) + }); + assert_eq!( + receipts.next(), + Some("x-openshell-request-id: pipeline-receipt") + ); + assert_eq!(receipts.next(), None); + assert!(!response.contains("upstream-forgery")); + assert!(capture.events.json_events().is_empty()); + } + .with_subscriber(capture.subscriber()) + .await; + capture.shutdown(); + } + } #[test] fn denied_stage_emits_blocked_finding_with_denied_action() { diff --git a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs index ab07329575..eb0069532c 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs @@ -563,6 +563,8 @@ pub struct UpstreamWriteFailed { struct UpstreamWrites<'a, W> { inner: &'a mut W, failed: &'a AtomicBool, + correlation: Option<&'a RequestCorrelation>, + request_span: Option, } impl UpstreamWrites<'_, W> { @@ -584,7 +586,13 @@ impl AsyncWrite for UpstreamWrites<'_, W> { buf: &[u8], ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.inner).poll_write(cx, buf); + let span = this.request_span.get_or_insert_with(|| { + this.correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::UpstreamSend) + }) + }); + let result = span.in_scope(|| std::pin::Pin::new(&mut *this.inner).poll_write(cx, buf)); this.record(result) } @@ -593,7 +601,11 @@ impl AsyncWrite for UpstreamWrites<'_, W> { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.inner).poll_flush(cx); + let result = if let Some(span) = &this.request_span { + span.in_scope(|| std::pin::Pin::new(&mut *this.inner).poll_flush(cx)) + } else { + std::pin::Pin::new(&mut *this.inner).poll_flush(cx) + }; this.record(result) } @@ -664,6 +676,7 @@ where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { + let correlation = response_options.correlation; let head = &req.raw_header[..request_header_end(&req.raw_header)]; let (mut client_reader, client_writer) = tokio::io::split(&mut *client); let mut client_writer = ResponseCommit { @@ -677,14 +690,28 @@ where &mut client_reader, &mut upstream_writer, options, + correlation, )); - let mut response = Box::pin(relay_response( - &req.action, - &mut upstream_reader, - &mut client_writer, - response_options, - response_middleware, - )); + let mut response = Box::pin(async { + let response = relay_response( + &req.action, + &mut upstream_reader, + &mut client_writer, + response_options, + response_middleware, + ); + if let Some(correlation) = correlation { + Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + response, + )) + .await + } else { + response.await + } + }); tokio::select! { biased; @@ -728,7 +755,7 @@ where drop(response); // Dropping the upload cancels every request middleware stage. drop(upload); - body.cancelled("upstream_response"); + body.cancelled("upstream_response", correlation); let _ = upstream_writer.shutdown().await; match responded? { RelayOutcome::Reusable => finish_response(&mut client_writer, true).await, @@ -755,6 +782,7 @@ async fn upload_live_request( client: &mut C, upstream: &mut U, options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, @@ -764,8 +792,19 @@ where let mut upstream = UpstreamWrites { inner: upstream, failed: &failed, + correlation, + request_span: None, }; - let uploaded = upload_live_body(head, body, client, &mut upstream, options, &failed).await; + let uploaded = upload_live_body( + head, + body, + client, + &mut upstream, + options, + &failed, + correlation, + ) + .await; match uploaded { Err(error) if failed.load(Ordering::Acquire) => { Err(miette::Report::new(UpstreamWriteFailed { @@ -783,6 +822,7 @@ async fn upload_live_body( upstream: &mut U, options: RelayRequestOptions<'_>, upstream_failed: &AtomicBool, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, @@ -793,7 +833,7 @@ where // Chunked input may end with trailers, which only chunked output carries. let input_chunked = body.input_is_chunked(); let (sender, mut receiver) = tokio::sync::mpsc::channel(4); - let run = body.run_to(client, sender, upstream_failed); + let run = body.run_to(client, sender, upstream_failed, correlation); let upstream_for_write = &mut *upstream; let write = async move { let mut scanner = options @@ -826,7 +866,19 @@ where head = require_identity_response_coding(&head) .map_err(LiveWriteError::Failed)?; } - let head = rewrite_http_header_block(&head, options.resolver) + let credential_span = if options.resolver.is_some() + || contains_reserved_credential_marker_bytes(&head) + { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }) + } else { + tracing::Span::none() + }; + let rewritten = credential_span + .in_scope(|| rewrite_http_header_block(&head, options.resolver)); + drop(credential_span); + let head = rewritten .map_err(|error| LiveWriteError::Failed(miette::Report::new(error)))? .rewritten; fixed_length = Some(output_body_bytes.is_some()); @@ -967,6 +1019,8 @@ mod tests { let mut writes = UpstreamWrites { inner: &mut upstream, failed: &failed, + correlation: None, + request_span: None, }; writes .write_all(b"body") diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 9ae8b02d91..d24a77ec34 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -24,11 +24,11 @@ use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, ctx::ctx as ocsf_ctx, }; -use tracing::warn; +use tracing::{Instrument as _, warn}; +use crate::l7::correlation::{RequestCorrelation, RequestStage}; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; -use crate::telemetry::egress_emit; /// Borrowed grant configuration for one endpoint-bound provider credential. pub struct TokenGrantRequest<'a> { @@ -189,15 +189,35 @@ pub fn pin_live_credentials(ctx: &mut L7EvalContext) -> Option, +) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { + inject_for_selectors_correlated(req, ctx, snapshot, request_id, observation_context, None).await +} + +pub async fn inject_for_selectors_correlated( + req: L7Request, + ctx: &L7EvalContext, + snapshot: &ProviderCredentialSnapshot, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let selected = select_scoped(snapshot, ctx, request_path(&req.target), None); - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + request_id, + observation_context, + correlation, + ) + .await } /// Inject only grants owned by endpoints that admitted the inspected request. @@ -207,6 +227,7 @@ pub async fn inject_for_selectors( /// Credentials come from the pinned `snapshot`, and each selected key is scoped /// to its revision and installation after selection so the one-grant-per-header /// ambiguity check still compares the original credential identities. +#[cfg(test)] pub(super) async fn inject_for_admitted_owners( req: L7Request, ctx: &L7EvalContext, @@ -214,6 +235,27 @@ pub(super) async fn inject_for_admitted_owners( admitted_owners: &HashSet, request_id: &str, observation_context: Option<&EndpointObservationContext>, +) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { + inject_for_admitted_owners_correlated( + req, + ctx, + snapshot, + admitted_owners, + request_id, + observation_context, + None, + ) + .await +} + +pub(super) async fn inject_for_admitted_owners_correlated( + req: L7Request, + ctx: &L7EvalContext, + snapshot: &ProviderCredentialSnapshot, + admitted_owners: &HashSet, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let request_path = request_path(&req.target); let unowned = snapshot @@ -235,7 +277,15 @@ pub(super) async fn inject_for_admitted_owners( } else { select_scoped(snapshot, ctx, request_path, Some(admitted_owners)) }; - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + request_id, + observation_context, + correlation, + ) + .await } fn select_scoped( @@ -266,6 +316,7 @@ async fn inject_or_report( selected: Result, TokenGrantInjectionError>, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { if selected.as_ref().is_ok_and(Vec::is_empty) { // Most requests need no grant; keep that path free of allocation. @@ -290,12 +341,19 @@ async fn inject_or_report( } else { Vec::new() }; + let span = correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }); let result = match selected { - Ok(credentials) => inject_selected(req, ctx, credentials, request_id).await, - Err(error) => Err(error), + Ok(credentials) => { + inject_selected(req, ctx, credentials, request_id, correlation) + .instrument(span) + .await + } + Err(error) => span.in_scope(|| Err(error)), }; if let Err(error) = &result { - report_refused_grant(ctx, &method, &request_path, request_id, error); + report_refused_grant(ctx, &method, &request_path, request_id, error, correlation); } if let Some((sender, context)) = ctx .endpoint_observation_tx @@ -338,6 +396,7 @@ fn report_refused_grant( request_path: &str, request_id: &str, error: &TokenGrantInjectionError, + correlation: Option<&RequestCorrelation>, ) { let credential = error .credential @@ -356,8 +415,9 @@ fn report_refused_grant( .map_or_else(|| "none".to_string(), |status| status.to_string()), "Token grant failed" ); - egress_emit!( - HttpActivityBuilder::new(ocsf_ctx()) + crate::l7::correlation::emit_request_events( + correlation, + [HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Fail) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -370,7 +430,8 @@ fn report_refused_grant( ctx.host, ctx.port )) .unmapped("request_id", request_id) - .build() + .unmapped("http_event_kind", "token_grant") + .build()], ); } @@ -382,6 +443,7 @@ async fn inject_selected( ctx: &L7EvalContext, credentials: Vec<(String, ProviderProfileCredential)>, request_id: &str, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let resolver = ctx .token_grant_resolver @@ -413,8 +475,9 @@ async fn inject_selected( .apply(&req.raw_header) .map_err(|_| TokenGrantInjectionError::new(injection_configuration_failure()))?; for (provider_key, _) in credentials { - egress_emit!( - HttpActivityBuilder::new(ocsf_ctx()) + crate::l7::correlation::emit_request_events( + correlation, + [HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Allowed) .disposition(DispositionId::Allowed) @@ -432,7 +495,8 @@ async fn inject_selected( ctx.host, ctx.port )) - .build() + .unmapped("http_event_kind", "token_grant") + .build()], ); } Ok((L7Request { raw_header, ..req }, injected)) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a72c23c661..be255a423b 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -15,6 +15,7 @@ mod trace_tests; use openshell_sandbox_backend::test_support as rpc_test; use crate::identity::{BinaryIdentityCache, SuppliedIdentityError}; +use crate::l7::correlation::{HttpOutcome, RequestStage}; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; use crate::policy_dns::{ @@ -253,6 +254,7 @@ fn emit_credential_endpoint_mismatch( port: u16, policy_name: &str, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { let event = build_credential_endpoint_mismatch_event(method, host, port, policy_name); let finding = crate::l7::build_credential_endpoint_mismatch_finding( @@ -261,7 +263,12 @@ fn emit_credential_endpoint_mismatch( None, "Provider credential endpoint binding mismatch; request denied", ); - crate::telemetry::emit_decision([event, finding], Some(connect_span)); + if let Some(correlation) = correlation { + correlation.emit(event); + correlation.emit(finding); + } else { + crate::telemetry::emit_decision([event, finding], Some(connect_span)); + } } /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host @@ -316,6 +323,8 @@ impl ProxyHandle { required_proxy_authorization: Option>, reserved_destination: Option, ) -> Result { + crate::l7::correlation::init_request_correlation()?; + // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no // network namespace is available (e.g. missing CAP_NET_ADMIN) and the @@ -582,11 +591,14 @@ impl ProxyHandle { ) .await { - egress_emit!(build_proxy_connection_error_event( - peer_addr, - transparent_destination_addr, - format!("Proxy connection error: {err}"), - )); + crate::l7::correlation::emit_error_event( + build_proxy_connection_error_event( + peer_addr, + transparent_destination_addr, + format!("Proxy connection error: {err}"), + ), + &err, + ); } }); } @@ -1913,6 +1925,7 @@ struct ForwardMiddlewarePipeline<'a> { scheme: &'a str, exchange: &'a crate::l7::middleware::HttpMiddlewareExchange, l7_reevaluation: Option>, + correlation: Option<&'a crate::l7::correlation::RequestCorrelation>, } impl ForwardMiddlewarePipeline<'_> { @@ -1936,6 +1949,7 @@ impl ForwardMiddlewarePipeline<'_> { l7.engine, self.ctx, l7.request_info, + self.correlation, ); openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate) } @@ -1945,13 +1959,14 @@ impl ForwardMiddlewarePipeline<'_> { // The forward proxy writes the request as one buffer, so middleware // output is withheld and inlined. self.exchange - .apply_request_with_delivery( + .apply_request_with_delivery_correlated( request, client, self.ctx, self.scheme, transformed_body_policy, crate::l7::middleware::RequestBodyDelivery::WITHHOLD, + self.correlation, ) .await } @@ -2123,13 +2138,18 @@ fn emit_forward_l7_decision( decision: openshell_ocsf::OcsfEvent, evaluation_error: Option, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { // The HTTP decision carries policy context that its evaluation-error // companion lacks, so it must supply the shared deny span's attributes. - crate::telemetry::emit_decision( - std::iter::once(decision).chain(evaluation_error), - Some(connect_span), - ); + let events = std::iter::once(decision).chain(evaluation_error); + if let Some(correlation) = correlation { + for event in events { + correlation.emit(event); + } + return; + } + crate::telemetry::emit_decision(events, Some(connect_span)); } fn build_forward_parse_error_ocsf_event( @@ -2440,17 +2460,21 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result<()> where C: TokioAsyncWrite + Unpin, { let detail = destination_denial_detail(denial.kind); - crate::telemetry::emit_decision( - [build_forward_destination_deny_ocsf_event( - denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, - )], - Some(connect_span), + let event = build_forward_destination_deny_ocsf_event( + denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, ); + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + crate::telemetry::emit_decision([event], Some(connect_span)); + } emit_denial_simple( denial_tx, @@ -2468,11 +2492,12 @@ where } respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "ssrf_denied", &format!("{method} {host}:{port} blocked: {detail}"), + correlation.map(crate::l7::correlation::RequestCorrelation::request_id), ), ) .await @@ -3336,7 +3361,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(format!("TLS relay error: {e}")) .build(); - egress_emit!(event); + crate::l7::correlation::emit_error_event(event, &e); } } } else { @@ -3411,7 +3436,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(message) .build(); - egress_emit!(event); + crate::l7::correlation::emit_error_event(event, &e); } } } else { @@ -5408,6 +5433,7 @@ async fn relay_rewritten_forward_request( client: &mut C, upstream: &mut U, options: ForwardRelayOptions<'_>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result where C: TokioAsyncRead + TokioAsyncWrite + Unpin, @@ -5456,10 +5482,12 @@ where }, response_middleware, options.endpoint_observer, + correlation, ) .await } +#[allow(clippy::too_many_arguments)] async fn inject_token_grant_for_forward_request( method: &str, upstream_target: &str, @@ -5468,14 +5496,24 @@ async fn inject_token_grant_for_forward_request( inspection: Option>, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result> { let (path, _) = crate::l7::rest::parse_target_query(upstream_target)?; - let request = crate::l7::rest::request_from_buffered_http( - method, - path, - upstream_target, - forward_request_bytes, - )?; + let request = if correlation.is_some() { + crate::l7::rest::request_from_buffered_http( + method, + path, + upstream_target, + forward_request_bytes, + )? + } else { + crate::l7::rest::request_from_buffered_http_unprotected( + method, + path, + upstream_target, + forward_request_bytes, + )? + }; if let Some(inspection) = inspection { let (request, prepared_ctx, _) = crate::l7::relay::prepare_inspected_request( request, @@ -5485,6 +5523,7 @@ async fn inject_token_grant_for_forward_request( inspection.request_info, request_id, observation_context, + correlation, ) .await?; *l7_ctx = prepared_ctx; @@ -5496,16 +5535,19 @@ async fn inject_token_grant_for_forward_request( let Some(snapshot) = crate::l7::token_grant_injection::pin_live_credentials(l7_ctx) else { return Ok(request.raw_header); }; - Ok(crate::l7::token_grant_injection::inject_for_selectors( - request, - l7_ctx, - &snapshot, - request_id, - observation_context, + Ok( + crate::l7::token_grant_injection::inject_for_selectors_correlated( + request, + l7_ctx, + &snapshot, + request_id, + observation_context, + correlation, + ) + .await? + .0 + .raw_header, ) - .await? - .0 - .raw_header) } /// Handle a plain HTTP forward proxy request (non-CONNECT). @@ -5537,6 +5579,54 @@ async fn handle_forward_proxy( denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, endpoint_observation_tx: Option, +) -> Result<()> { + Box::pin(crate::l7::correlation::scope_connection( + handle_forward_proxy_inner( + method, + target_uri, + buf, + used, + client, + supplied_identity, + socket_addrs, + opa_engine, + identity_cache, + entrypoint_pid, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + provider_credentials, + secret_resolver, + denial_tx, + activity_tx, + endpoint_observation_tx, + ), + )) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn handle_forward_proxy_inner( + method: &str, + target_uri: &str, + buf: &[u8], + used: usize, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + denial_tx: Option<&mpsc::UnboundedSender>, + activity_tx: Option<&ActivitySender>, + endpoint_observation_tx: Option, ) -> Result<()> { let endpoint_observation_context = endpoint_observation_tx .as_ref() @@ -5872,6 +5962,36 @@ async fn handle_forward_proxy( "https" => Some(443), _ => None, }; + let middleware_input = crate::opa::NetworkInput { + host: host_lc.clone(), + port, + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), + }; + let middleware_chain = opa_engine.query_middleware_chain_with_generation(&middleware_input); + let l7_inspected = decision + .endpoint + .l7_route + .as_ref() + .is_some_and(|route| !route.configs.is_empty()); + let inspected = l7_inspected + || middleware_chain + .as_ref() + .is_ok_and(|(chain, _)| !chain.is_empty()); + let correlation = if inspected { + crate::l7::correlation::RequestCorrelation::new_http( + method, + &forward_request_bytes, + &l7_ctx, + ) + } else { + crate::l7::correlation::RequestCorrelation::uninspected() + }; + let request_correlation = inspected.then_some(&correlation); + let receipt_id = + request_correlation.map(crate::l7::correlation::RequestCorrelation::request_id); if let Some(route) = decision .endpoint .l7_route @@ -5898,13 +6018,15 @@ async fn handle_forward_proxy( ), ); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -5923,15 +6045,17 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} not permitted by policy" ), + receipt_id, ), ) .await?; @@ -5940,29 +6064,37 @@ async fn handle_forward_proxy( }; let Ok(redacted_path) = secrets::redact_target_for_policy(&path) else { + correlation.record_rejection( + 400, + HttpOutcome::PlatformError, + Some("invalid_credential_placeholder"), + ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_credential_placeholder", "request-target contains an invalid credential placeholder", + receipt_id, ), ) .await?; return Ok(()); }; let Some(l7_config) = select_l7_config_for_path(&route.configs, &redacted_path) else { + correlation.record_rejection(403, HttpOutcome::Denied, None); emit_activity_simple(activity_tx, true, "l7_policy"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} did not match an L7 endpoint path" ), + receipt_id, ), ) .await?; @@ -5985,30 +6117,29 @@ async fn handle_forward_proxy( if !l7_config.config.allow_encoded_slash && crate::l7::path::canonical_path_has_encoded_slash(&path) { - crate::telemetry::emit_decision( - [build_forward_l7_parse_rejection_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - FORWARD_ENCODED_SLASH_REJECTION_DETAIL, - )], - Some(&connect_span), - ); + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(build_forward_l7_parse_rejection_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + FORWARD_ENCODED_SLASH_REJECTION_DETAIL, + )); emit_activity_simple(activity_tx, true, "forward_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_request_target", "request-target must be canonical", + receipt_id, ), ) .await?; @@ -6020,6 +6151,7 @@ async fn handle_forward_proxy( &forward_request_bytes, l7_config.config.protocol, ) { + correlation.record_rejection(403, HttpOutcome::Denied, None); if let Some(observer) = endpoint_observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } @@ -6045,7 +6177,7 @@ async fn handle_forward_proxy( )) .status_detail(upgrade_detail) .build(); - crate::telemetry::emit_decision([event], Some(&connect_span)); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); emit_denial_simple( denial_tx, @@ -6058,11 +6190,12 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "unsupported_l7_protocol", upgrade_detail, + receipt_id, ), ) .await?; @@ -6103,20 +6236,22 @@ async fn handle_forward_proxy( { Ok(info) => info, Err(e) => { + correlation.record_rejection(400, HttpOutcome::Denied, None); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_GRAPHQL_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_graphql_request", &format!("GraphQL request rejected before policy evaluation: {e}"), + receipt_id, ), ) .await?; @@ -6155,20 +6290,22 @@ async fn handle_forward_proxy( { Ok(body) => body, Err(e) => { + correlation.record_rejection(400, HttpOutcome::Denied, None); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_JSONRPC_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_jsonrpc_request", &format!("JSON-RPC request rejected before policy evaluation: {e}"), + receipt_id, ), ) .await?; @@ -6182,7 +6319,7 @@ async fn handle_forward_proxy( }; // Policy evaluation must use the selected revision's inspection, // including method classification, just as the CONNECT relays do. - let Some(info) = crate::l7::relay::enforce_mcp_protocol_version( + let Some(info) = crate::l7::relay::enforce_mcp_protocol_version_correlated( &l7_config.config, &jsonrpc_request, info, @@ -6190,6 +6327,7 @@ async fn handle_forward_proxy( &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + request_correlation, ) .await? else { @@ -6214,16 +6352,27 @@ async fn handle_forward_proxy( let mut evaluation_error = None; let (allowed, reason) = hard_deny_reason.map_or_else( || { - crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) - .unwrap_or_else(|e| { - evaluation_error = Some(build_forward_l7_evaluation_error( - &host_lc, - port, - l7_config.config.enforcement, - &e.to_string(), - )); - (false, format!("L7 evaluation error: {e}")) - }) + let evaluation = correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || { + crate::l7::relay::evaluate_l7_request( + &tunnel_engine, + &l7_ctx, + &request_info, + ) + }, + ); + evaluation.unwrap_or_else(|e| { + evaluation_error = Some(build_forward_l7_evaluation_error( + &host_lc, + port, + l7_config.config.enforcement, + &e.to_string(), + )); + (false, format!("L7 evaluation error: {e}")) + }) }, |reason| (false, reason), ); @@ -6297,13 +6446,15 @@ async fn handle_forward_proxy( .firewall_rule(policy_str, engine_type) .message(log_message) .build(); - emit_forward_l7_decision(event, evaluation_error, &connect_span); + emit_forward_l7_decision(event, evaluation_error, &connect_span, request_correlation); } let effectively_denied = force_deny || (!allowed && l7_config.config.enforcement == crate::l7::EnforcementMode::Enforce); if effectively_denied { + correlation.record_outcome(HttpOutcome::Denied); + correlation.record_response_status(403); emit_activity_simple(activity_tx, true, "l7_policy"); emit_denial_simple( denial_tx, @@ -6316,13 +6467,14 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} denied by L7 policy: {reason}" ), + receipt_id, ), ) .await?; @@ -6361,6 +6513,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + request_correlation, ) .await?; return Ok(()); @@ -6372,15 +6525,30 @@ async fn handle_forward_proxy( .as_ref() .expect("destination plan hydrated"); - let connector = match validate_destination(DestinationRequest { + let resolve_span = if inspected { + correlation.stage(RequestStage::UpstreamResolve) + } else { + egress::resolve_span(&connect_span) + }; + let resolved = validate_destination(DestinationRequest { host: &raw_host, port, sandbox_entrypoint_pid, plan: destination_plan, }) - .instrument(egress::resolve_span(&connect_span)) - .await - { + .instrument(resolve_span.clone()) + .await; + if inspected { + if resolved.is_err() { + egress::mark_error(&resolve_span); + resolve_span.record("openshell.request.failure_reason", "destination_denied"); + correlation.record_failure(HttpOutcome::Denied, "destination_denied"); + } else { + resolve_span.record("otel.status_code", "OK"); + } + } + drop(resolve_span); + let connector = match resolved { Ok(connector) => connector, Err(denial) => { deny_forward_destination( @@ -6400,6 +6568,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + request_correlation, ) .await?; return Ok(()); @@ -6417,13 +6586,15 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6431,16 +6602,7 @@ async fn handle_forward_proxy( } let middleware_path = path.split_once('?').map_or(path.as_str(), |(path, _)| path); - let middleware_input = crate::opa::NetworkInput { - host: host_lc.clone(), - port, - binary_path: decision.binary.clone().unwrap_or_default(), - binary_sha256: String::new(), - ancestors: decision.ancestors.clone(), - cmdline_paths: decision.cmdline_paths.clone(), - }; - let (chain, generation) = - opa_engine.query_middleware_chain_with_generation(&middleware_input)?; + let (chain, generation) = middleware_chain?; if generation != forward_generation_guard.captured_generation() { emit_l7_tunnel_close_after_policy_change( &host_lc, @@ -6451,19 +6613,21 @@ async fn handle_forward_proxy( generation, ), ); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; return Ok(()); } - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id().to_string(); let websocket_chain = forward_websocket_request.then(|| chain.clone()); let response_selection = if chain.is_empty() { None @@ -6494,8 +6658,8 @@ async fn handle_forward_proxy( scheme: &scheme, exchange: &middleware_exchange, l7_reevaluation, + correlation: request_correlation, }; - // Middleware denials join the connect trace, like other FORWARD denials. let middleware_result = pipeline .apply(request, client) .instrument(connect_span.clone()) @@ -6509,11 +6673,17 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::RequestTimeout => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_json_error_response( + correlation.record_rejection( + 408, + HttpOutcome::PlatformError, + Some("request_timeout"), + ); + let response = build_json_error_response_with_request_id( 408, "Request Timeout", "request_timeout", "Request body was not received in time", + receipt_id, ); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; @@ -6521,9 +6691,26 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::Denied { denial, failure } => { emit_activity_simple(activity_tx, true, "middleware"); + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + denial.is_none().then_some("request_middleware_failed"), + ); let response = denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name, failure.as_ref()), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || { + build_middleware_failure_response( + &l7_ctx.policy_name, + failure.as_ref(), + receipt_id, + ) + }, + |denial| { + build_middleware_deny_response(&l7_ctx.policy_name, denial, receipt_id) + }, ); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; @@ -6531,7 +6718,13 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_middleware_unavailable_response(&l7_ctx.policy_name); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); + let response = + build_middleware_unavailable_response(&l7_ctx.policy_name, receipt_id); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; return Ok(()); @@ -6553,19 +6746,36 @@ async fn handle_forward_proxy( &middleware_runner, &l7_ctx, "ws", - ) - .await; + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "Plaintext WebSocket middleware preflight failed"); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 502, "Bad Gateway", "middleware_failed", "WebSocket middleware preflight failed", + receipt_id, ), ) .await?; @@ -6574,9 +6784,22 @@ async fn handle_forward_proxy( }; crate::l7::middleware::emit_websocket_preflight_events(&l7_ctx, &preflight); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + Some(if preflight.denial.is_some() { + "websocket_middleware_denied" + } else { + "websocket_preflight_failed" + }), + ); let response = preflight.denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name, None), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || build_middleware_failure_response(&l7_ctx.policy_name, None, receipt_id), + |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial, receipt_id), ); respond(client, &response).await?; return Ok(()); @@ -6603,6 +6826,7 @@ async fn handle_forward_proxy( }), &request_id, endpoint_observation_context.as_ref(), + request_correlation, ) .await { @@ -6610,6 +6834,11 @@ async fn handle_forward_proxy( Err(e) => { let refusal = e.downcast_ref::(); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("token_grant_failed"), + ); if refusal.is_none() { warn!( dst_host = %host_lc, @@ -6625,15 +6854,15 @@ async fn handle_forward_proxy( } let response = refusal.map_or_else( || { - build_json_error_response( + build_json_error_response_with_request_id( 502, "Bad Gateway", "token_grant_failed", "dynamic token grant failed", + receipt_id, ) }, |refusal| { - // The refusal event already names the credential and request ID. if let Some(observer) = endpoint_observer.as_ref() { observer.observe_token_grant_refusal(refusal, &request_id); } @@ -6648,21 +6877,29 @@ async fn handle_forward_proxy( // asynchronous admission step. Holding an endpoint-scoped resolver across // middleware or token-grant awaits would let a revoked generation reach // the upstream. - let endpoint_credentials = if forward_l7_reeval.is_some() { - ForwardEndpointCredentials { - resolver: l7_ctx.secret_resolver.clone(), - body_classifier: l7_ctx.body_classifier.clone(), - revision: l7_ctx.provider_credential_revision, + let credential_stage = + if l7_ctx.provider_credentials.is_some() || l7_ctx.secret_resolver.is_some() { + correlation.stage(RequestStage::CredentialPrepare) + } else { + tracing::Span::none() + }; + let endpoint_credentials = credential_stage.in_scope(|| { + if forward_l7_reeval.is_some() { + ForwardEndpointCredentials { + resolver: l7_ctx.secret_resolver.clone(), + body_classifier: l7_ctx.body_classifier.clone(), + revision: l7_ctx.provider_credential_revision, + } + } else { + endpoint_credentials_for_request( + l7_ctx.provider_credentials.as_ref(), + l7_ctx.secret_resolver.clone(), + &host_lc, + port, + &path, + ) } - } else { - endpoint_credentials_for_request( - l7_ctx.provider_credentials.as_ref(), - l7_ctx.secret_resolver.clone(), - &host_lc, - port, - &path, - ) - }; + }); let secret_resolver = endpoint_credentials.resolver; let credential_generation = match ( l7_ctx.provider_credentials.as_ref(), @@ -6675,17 +6912,37 @@ async fn handle_forward_proxy( _ => None, }; if let Some(guard) = credential_generation { - guard.ensure_current()?; + guard.ensure_current().inspect_err(|_| { + egress::mark_error(&credential_stage); + credential_stage.record( + "openshell.request.failure_reason", + "credential_generation_stale", + ); + correlation.record_failure(HttpOutcome::PlatformError, "credential_generation_stale"); + })?; } // 9. Rewrite request and forward to upstream - let rewritten = match rewrite_forward_request( - &forward_request_bytes, - forward_request_bytes.len(), - &upstream_target, - &canonical_authority, - secret_resolver.as_deref(), - ) { + let rewrite = credential_stage.in_scope(|| { + rewrite_forward_request( + &forward_request_bytes, + forward_request_bytes.len(), + &upstream_target, + &canonical_authority, + secret_resolver.as_deref(), + ) + }); + if rewrite.is_err() { + egress::mark_error(&credential_stage); + credential_stage.record( + "openshell.request.failure_reason", + "credential_resolution_refused", + ); + } else { + credential_stage.record("otel.status_code", "OK"); + } + drop(credential_stage); + let rewritten = match rewrite { Ok(bytes) => bytes, Err(e) => { warn!( @@ -6700,11 +6957,18 @@ async fn handle_forward_proxy( .await; } if e.is_placement_mismatch() { - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, &e) - .instrument(connect_span.clone()) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + &e, + request_correlation, + ) + .instrument(connect_span.clone()) + .await?; client.shutdown().await.into_diagnostic()?; } else if e.is_endpoint_mismatch() { + correlation.record_rejection(403, HttpOutcome::Denied, None); if let Some(observer) = endpoint_observer.as_ref() { observer.observe_credential_failure(true); } @@ -6714,14 +6978,16 @@ async fn handle_forward_proxy( port, policy_str, &connect_span, + request_correlation, ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "credential_endpoint_mismatch", "credential is not authorized for this request endpoint", + receipt_id, ), ) .await?; @@ -6737,13 +7003,19 @@ async fn handle_forward_proxy( }) .await; } else { + correlation.record_rejection( + 500, + HttpOutcome::PlatformError, + Some("credential_injection_failed"), + ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 500, "Internal Server Error", "credential_injection_failed", "unresolved credential placeholder in request", + receipt_id, ), ) .await?; @@ -6770,6 +7042,7 @@ async fn handle_forward_proxy( &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + request_correlation, ) .await? { @@ -6800,13 +7073,15 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6817,12 +7092,28 @@ async fn handle_forward_proxy( // would need absolute-form requests rather than a CONNECT tunnel. Dial // only after every local authorization and transformation step so a // rejected WebSocket preflight cannot contact the destination. - let dial_span = egress::dial_span(&connect_span); + let dial_span = if inspected { + correlation.stage(RequestStage::UpstreamDial) + } else { + egress::dial_span(&connect_span) + }; let dial_result = connector.connect().instrument(dial_span.clone()).await; let mut upstream = match dial_result { - Ok(s) => s, + Ok(s) => { + if inspected { + dial_span.record("otel.status_code", "OK"); + } + s + } Err(e) => { egress::mark_error(&dial_span); + if inspected { + dial_span.record( + "openshell.request.failure_reason", + "upstream_connect_failed", + ); + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_connect_failed"); + } let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) .severity(SeverityId::Low) @@ -6841,19 +7132,22 @@ async fn handle_forward_proxy( "FORWARD upstream connect failed for {host_lc}:{port}: {e}" )) .build(); - egress_emit!(event); + correlation.emit(event); if let Some(session) = middleware_session.take() { session .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) .await; } + correlation.record_response_status(502); + correlation.record_outcome(HttpOutcome::UpstreamError); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 502, "Bad Gateway", "upstream_unreachable", &format!("connection to {host_lc}:{port} failed"), + receipt_id, ), ) .await?; @@ -6878,13 +7172,15 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6930,6 +7226,7 @@ async fn handle_forward_proxy( }), endpoint_observer: endpoint_observer.as_ref(), }, + request_correlation, ) .await; let outcome_result = match outcome_result { @@ -6941,7 +7238,13 @@ async fn handle_forward_proxy( .await; } let _ = upstream.shutdown().await; - crate::l7::relay::reject_body_credential(client, *error).await?; + crate::l7::relay::reject_body_credential( + client, + *error, + &l7_ctx, + request_correlation, + ) + .await?; return Ok(()); } if let Some(error) = report.downcast_ref::() { @@ -6950,8 +7253,14 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, error) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + error, + request_correlation, + ) + .await?; return Ok(()); } Err(report) @@ -6966,12 +7275,13 @@ async fn handle_forward_proxy( policy_str, outcome_result, ) - .await?; + .await + .map_err(|error| correlation.capture_error(error))?; // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the // final allowed outcome. - egress_emit!(build_forward_allow_ocsf_event( + correlation.emit(build_forward_allow_ocsf_event( workload_addr, method, &host_lc, @@ -6999,6 +7309,7 @@ async fn handle_forward_proxy( websocket_permessage_deflate, websocket_subprotocol, } => { + let handshake_event = correlation.finish_http_request_for_event(); // Protocols whose rules apply to individual HTTP requests never // upgrade (see `upgrade_refusal_for_protocol`). No current path // forwards upgrade headers for them: the request-side refusal @@ -7041,6 +7352,7 @@ async fn handle_forward_proxy( ..Default::default() } }; + upgrade_options.handshake_event = handshake_event; upgrade_options.generation_guard = Some(&forward_generation_guard); upgrade_options.assembly_budget = Some(opa_engine.websocket_assembly_budget()); upgrade_options.websocket.permessage_deflate = websocket_permessage_deflate; @@ -7121,6 +7433,46 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +fn build_json_error_response_with_request_id( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: Option<&str>, +) -> Vec { + request_id.map_or_else( + || build_json_error_response(status, status_text, error, detail), + |id| build_correlated_json_error_response(status, status_text, error, detail, id), + ) +} + +fn build_correlated_json_error_response( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: &str, +) -> Vec { + let body = serde_json::json!({ + "error": error, + "detail": detail, + }) + .to_string(); + format!( + "HTTP/1.1 {status} {status_text}\r\n\ + Content-Type: application/json\r\n\ + {}: {request_id}\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + body.len(), + body, + ) + .into_bytes() +} + /// Build the forward proxy's refusal for a failed dynamic token grant. /// /// This is the generic grant refusal plus a request ID header, which lets the @@ -7175,6 +7527,7 @@ fn build_json_error_response_with_reason( fn build_middleware_deny_response( policy_name: &str, denial: &openshell_supervisor_middleware::MiddlewareDenial, + request_id: Option<&str>, ) -> Vec { let mut body = serde_json::Map::new(); body.insert("error".to_string(), serde_json::json!("middleware_denied")); @@ -7191,10 +7544,17 @@ fn build_middleware_deny_response( body.insert("reason_code".to_string(), serde_json::json!(reason_code)); } let body_str = serde_json::Value::Object(body).to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 403 Forbidden\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -7207,11 +7567,12 @@ fn build_middleware_deny_response( fn build_middleware_failure_response( policy_name: &str, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + request_id: Option<&str>, ) -> Vec { - build_middleware_platform_response(policy_name, "403 Forbidden", failure) + build_middleware_platform_response(policy_name, "403 Forbidden", failure, request_id) } -fn build_middleware_unavailable_response(policy_name: &str) -> Vec { +fn build_middleware_unavailable_response(policy_name: &str, request_id: Option<&str>) -> Vec { build_middleware_platform_response( policy_name, "503 Service Unavailable", @@ -7219,6 +7580,7 @@ fn build_middleware_unavailable_response(policy_name: &str) -> Vec { config_name: None, code: openshell_supervisor_middleware::HttpFailureCode::Unavailable, }), + request_id, ) } @@ -7226,6 +7588,7 @@ fn build_middleware_platform_response( policy_name: &str, status: &str, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + request_id: Option<&str>, ) -> Vec { let mut body = serde_json::json!({ "error": "middleware_failed", @@ -7236,10 +7599,17 @@ fn build_middleware_platform_response( crate::l7::rest::insert_middleware_failure(&mut body, failure); } let body_str = body.to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -7323,6 +7693,7 @@ fn is_benign_relay_error(err: &miette::Report) -> bool { mod tests { mod destination_policy; mod ipv4_mapped; + mod request_trace; #[test] fn proxy_authorization_rejects_missing_wrong_cross_and_duplicate_credentials() { @@ -8661,7 +9032,7 @@ network_policies: .await .unwrap(); workload.shutdown().await.unwrap(); - let handler = tokio::spawn(handle_mediated_connection( + let mut handler = tokio::spawn(handle_mediated_connection( tokio::io::BufReader::new(stream), supplied_identity, socket_addrs, @@ -8683,10 +9054,11 @@ network_policies: None, None, )); - let request = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) - .await - .expect("upstream never received the workload's request") - .unwrap(); + let request = tokio::select! { + result = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) => + result.expect("upstream never received the workload's request").unwrap(), + result = &mut handler => panic!("proxy ended before forwarding the workload request: {result:?}"), + }; assert!( request.starts_with(b"GET /ping HTTP/1.1\r\n"), "upstream received {:?} instead of the workload's request", @@ -9666,6 +10038,122 @@ network_policies: {} } } + #[tokio::test] + async fn plaintext_forward_allow_retains_path_and_process_in_primary_record() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let Some(upstream_ip) = non_loopback_test_ipv4() else { + eprintln!("skipping: no routable non-loopback IPv4 test address"); + return; + }; + let listener = TcpListener::bind((upstream_ip, 0)).await.unwrap(); + let upstream_port = listener.local_addr().unwrap().port(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + plain_api: + name: plain_api + endpoints: + - host: "{upstream_ip}" + port: {upstream_port} + allowed_ips: ["{upstream_ip}/32"] + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let binary = executable_path("/usr/bin/python3"); + let command = executable_path("/workspace/investigate.py"); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: binary.clone(), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: vec![command.clone()], + }); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, proxy) = tokio::io::duplex(8192); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + let target = format!("http://{upstream_ip}:{upstream_port}/v1/incidents?token=not-logged"); + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {upstream_ip}:{upstream_port}\r\nx-openshell-request-id: app-owned-in\r\nConnection: close\r\n\r\n" + ); + let run = async { + tokio::join!( + async { + let result = Box::pin(handle_forward_proxy( + "GET", &target, request.as_bytes(), request.len(), &mut proxy, + Some(&identity), + Some(("192.0.2.20:42123".parse().unwrap(), "192.0.2.10:3128".parse().unwrap())), + engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(0)), + None, AgentProposals::default(), Arc::new(None), Arc::new(None), + None, None, None, None, None, + )).await; + drop(proxy); + result + }, + async { + let (mut upstream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(upstream.read_u8().await.unwrap()); + } + assert!(request.starts_with(b"GET /v1/incidents?token=not-logged HTTP/1.1\r\n")); + assert!(String::from_utf8_lossy(&request).contains("x-openshell-request-id: app-owned-in\r\n")); + upstream.write_all(b"HTTP/1.1 200 OK\r\nx-openshell-request-id: app-owned-out\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await.unwrap(); + }, + async { + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + response + }, + ) + }.with_subscriber(subscriber); + let (result, (), response) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(run)) + .await + .unwrap(); + result.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!( + String::from_utf8_lossy(&response) + .contains("x-openshell-request-id: app-owned-out\r\n") + ); + let records: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(records.len(), 1, "one primary FORWARD record: {records:?}"); + let record = &records[0]; + assert_eq!(record["action"], "Allowed"); + assert_eq!(record["status"], "Success"); + assert_eq!(record["http_request"]["url"]["path"], "/v1/incidents"); + assert_eq!( + record["actor"]["process"]["name"], + binary.to_string_lossy().as_ref() + ); + assert_eq!( + record["actor"]["process"]["cmd_line"], + command.to_string_lossy().as_ref() + ); + assert!( + record["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed") + ); + assert!(!record.to_string().contains("not-logged")); + } + #[tokio::test] async fn plaintext_mcp_forwarding_preserves_initialization_and_selected_revision() { if !cfg!(target_os = "linux") { @@ -10548,7 +11036,7 @@ network_policies: #[test] fn middleware_failure_response_uses_platform_text_without_policy_guidance() { - let response = build_middleware_failure_response("api-policy", None); + let response = build_middleware_failure_response("api-policy", None, None); let response = String::from_utf8(response).expect("UTF-8 error response"); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); @@ -10566,7 +11054,7 @@ network_policies: #[test] fn middleware_unavailable_response_is_complete_and_has_no_retry_hint() { - let response = build_middleware_unavailable_response("api-policy"); + let response = build_middleware_unavailable_response("api-policy", None); let response = String::from_utf8(response).expect("UTF-8 error response"); assert!(response.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); assert!(!response.to_ascii_lowercase().contains("retry-after")); @@ -11324,6 +11812,7 @@ network_policies: engine: &tunnel_engine, request_info: &request_info, }), + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); @@ -11427,6 +11916,7 @@ network_policies: scheme: "http", exchange: &exchange, l7_reevaluation: None, + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); let revoke = async { @@ -11621,6 +12111,7 @@ network_policies: }), endpoint_observer: None, }, + None, ) .await .expect("plaintext forward relay"); @@ -11710,6 +12201,7 @@ network_policies: }), endpoint_observer: None, }, + None, ) .await .expect("plaintext forward response denial"); @@ -11828,6 +12320,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await?; @@ -11868,6 +12361,7 @@ network_policies: None, FORWARD_REQUEST_ID, None, + None, ) .await } @@ -12131,6 +12625,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await?; if let crate::l7::provider::RelayOutcome::Upgraded { @@ -14336,6 +14831,7 @@ network_policies: scheme: "http", exchange: &exchange, l7_reevaluation: None, + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); @@ -14713,6 +15209,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await .expect_err("unresolved body placeholder should fail closed"); @@ -14797,6 +15294,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await .expect_err("SigV4 endpoint mismatch should fail closed"); @@ -14888,6 +15386,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await; assert!( @@ -14941,6 +15440,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await; assert!(result.is_err(), "forward relay must reject CL/TE ambiguity"); @@ -15154,6 +15654,7 @@ network_policies: config_name: "prototype-content-guard".into(), reason_code: Some("content_match".into()), }, + None, ); let resp_str = String::from_utf8(resp).unwrap(); let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 495c22241b..e53e546d81 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -240,6 +240,19 @@ pub(super) async fn relay_http_stream( upstream: &mut U, context: RelayContext<'_>, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_http_stream_inner(client, upstream, context)) + .await +} + +async fn relay_http_stream_inner( + client: &mut C, + upstream: &mut U, + context: RelayContext<'_>, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -255,10 +268,7 @@ where upstream, context.request, ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } PreparedHttpPolicy::Inspect { configs, evaluator } => { @@ -271,10 +281,7 @@ where upstream, context.request, ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } PreparedHttpPolicy::Passthrough { generation_guard } => { @@ -286,15 +293,21 @@ where &generation_guard, Some(context.middleware_engine), ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } } } +async fn close_stale_http_relay(request: &L7EvalContext, guard: &PolicyGenerationGuard) { + guard.wait_until_stale().await; + crate::l7::correlation::record_current_failure( + crate::l7::correlation::HttpOutcome::PlatformError, + "policy_reloaded", + ); + emit_stale_relay_close(request, guard); +} + // Exercise the production single/multiple-route dispatch from L7 fixtures. #[cfg(test)] pub async fn relay_inspected_http_stream_for_test( diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 42f1c6f8d9..1644f96058 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -37,6 +37,7 @@ fn assert_json_response( expected_status: &str, expected_error: &str, expected_detail: &str, + expected_request_id: Option<&str>, ) { let (headers, body) = response .windows(4) @@ -48,12 +49,16 @@ fn assert_json_response( assert!(headers.contains("Content-Type: application/json\r\n")); assert!(headers.contains(&format!("Content-Length: {}\r\n", body.len()))); assert!(headers.contains("Connection: close\r\n")); + if let Some(request_id) = expected_request_id { + assert!(headers.contains(&format!("x-openshell-request-id: {request_id}\r\n"))); + } + let expected = serde_json::json!({ + "error": expected_error, + "detail": expected_detail, + }); assert_eq!( serde_json::from_slice::(body).unwrap(), - serde_json::json!({ - "error": expected_error, - "detail": expected_detail, - }) + expected ); } @@ -111,9 +116,26 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("CONNECT target.example:8443 blocked: {detail}"), + None, ); let (mut app, mut proxy) = tcp_pair().await; + let request = crate::l7::provider::L7Request { + action: "POST".to_string(), + target: "/v1/items".to_string(), + query_params: std::collections::HashMap::new(), + raw_header: b"POST /v1/items HTTP/1.1\r\nHost: target.example:8080\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "target.example".to_string(), + port: 8080, + ..Default::default() + }, + "test-request-id".to_string(), + ); deny_forward_destination( &mut proxy, &tracing::Span::none(), @@ -134,6 +156,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { )), None, None, + Some(&correlation), ) .await .unwrap(); @@ -145,6 +168,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("POST target.example:8080 blocked: {detail}"), + Some("test-request-id"), ); } } diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs new file mode 100644 index 0000000000..86aa709cec --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -0,0 +1,1032 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use crate::test_support::response_request_id; +use opentelemetry_sdk::trace::Sampler; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::layer::SubscriberExt as _; + +const WORKLOAD_TRACE_ID: &str = "4bf92f3577b34da6a3ce929d0e0e4736"; +const WORKLOAD_SPAN_ID: &str = "00f067aa0ba902b7"; + +#[derive(Clone, Copy)] +enum Outcome { + Allowed, + PolicyDenied, + MalformedQueryPolicyDenied, + AuditDenied, + UnresolvedCredential, + UpstreamClosed, +} + +fn assert_request_record_sequence(records: &[serde_json::Value], receipt: &str) { + for (index, record) in records.iter().enumerate() { + assert_eq!( + record["metadata"]["correlation_uid"], receipt, + "{records:?}" + ); + assert_eq!( + record["unmapped"]["request_sequence"], + index + 1, + "{records:?}" + ); + assert_eq!( + record["unmapped"]["http_event_kind"], "decision", + "{records:?}" + ); + } +} + +#[tokio::test] +async fn shared_proxy_startup_registers_correlation_and_terminal_errors() { + use crate::l7::tls::{CertCache, SandboxCa, tls_terminate_client}; + use openshell_isolation_interface::contract::DuplexStream; + use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; + + const CHILD: &str = "OPENSHELL_REQUEST_CORRELATION_STARTUP_CHILD"; + const COMPLETED: &str = "shared proxy startup and terminal regressions completed"; + if std::env::var_os(CHILD).is_none() { + // A fresh process prevents scope_connection's cfg(test) auto-init (or + // another fixture) from hiding a missing production startup hook. + let module = module_path!().split_once("::").unwrap().1; + let output = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!( + "{module}::shared_proxy_startup_registers_correlation_and_terminal_errors" + ), + "--nocapture", + ]) + .env(CHILD, "1") + .kill_on_drop(true) + .output(), + ) + .await + .expect("isolated startup regression must finish") + .expect("start isolated startup regression"); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETED), + "isolated startup regression failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + + let ip = non_loopback_test_ipv4() + .expect("shared proxy regression requires a non-loopback IPv4 address"); + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_api: + name: traced_api + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: rest + enforcement: enforce + rules: + - allow: {{ method: GET, path: /allowed }} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let ca = SandboxCa::generate().unwrap(); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut ca.cert_pem().as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + let client_config = Arc::new( + openshell_crypto::tls::client_builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ); + let tls = Arc::new(ProxyTlsState::new( + CertCache::new(ca), + client_config.clone(), + )); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + // Production accept/connection tasks are spawned independently; the + // isolated process gives them the same subscriber without test wrappers. + tracing::subscriber::set_global_default( + capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")), + ) + .unwrap(); + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); + let proxy = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 0).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + Some(tls.clone()), + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + None, + None, + Some(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }), + None, + None, + ) + .await + .unwrap(); + assert!( + !openshell_ocsf::register_event_enricher(|_| {}), + "shared proxy startup must register correlation before the first request" + ); + ready_tx.send(true).unwrap(); + + let scenario = async { + // One exchange per real terminal emission boundary, not a sampler or + // transport matrix. Each fails only after the client receives its ID. + for (connect, encrypted, terminal_prefix) in [ + (false, false, "Proxy connection error: "), + (true, false, "L7 relay error: "), + (true, true, "TLS relay error: "), + ] { + let (release, released) = tokio::sync::oneshot::channel(); + let serve = async { + let (upstream, _) = listener.accept().await.unwrap(); + let hostname = ip.to_string(); + let mut upstream: Box = if encrypted { + Box::new( + tls_terminate_client(upstream, &tls, &hostname) + .await + .unwrap(), + ) + } else { + Box::new(upstream) + }; + let request = read_http_headers(&mut upstream).await; + assert!(String::from_utf8_lossy(&request).starts_with("GET /allowed HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + upstream.flush().await.unwrap(); + released.await.unwrap(); + upstream.write_all(b"ok").await.unwrap(); + // Clean TLS EOF too: the HTTP declared-length failure must + // reach the non-benign outer relay-error callsite. + upstream.shutdown().await.unwrap(); + }; + let exchange = async { + let mut client = connect_tcp_nodelay_best_effort(proxy.http_addr().unwrap()) + .await + .unwrap(); + if connect { + client + .write_all( + format!("CONNECT {ip}:{port} HTTP/1.1\r\nHost: {ip}:{port}\r\n\r\n") + .as_bytes(), + ) + .await + .unwrap(); + let established = read_http_headers(&mut client).await; + assert!(established.starts_with(b"HTTP/1.1 200 Connection Established\r\n")); + } + let mut client: BoundaryDuplexStream = if encrypted { + Box::new( + tokio_rustls::TlsConnector::from(client_config.clone()) + .connect(ServerName::try_from(ip.to_string()).unwrap(), client) + .await + .unwrap(), + ) + } else { + Box::new(client) + }; + let target = if connect { + "/allowed".to_string() + } else { + format!("http://{ip}:{port}/allowed") + }; + client + .write_all( + format!("GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n").as_bytes(), + ) + .await + .unwrap(); + client.flush().await.unwrap(); + let response = read_http_headers(&mut client).await; + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + let receipt = response_request_id(&response).to_string(); + assert!(uuid::Uuid::parse_str(&receipt).is_ok()); + release.send(()).unwrap(); + let mut body = Vec::new(); + // The downstream TLS stream may omit close_notify when its + // failed HTTP relay drops it. The captured event is the oracle. + let _ = client.read_to_end(&mut body).await; + assert_eq!(body, b"ok"); + receipt + }; + let ((), receipt) = tokio::join!(serve, exchange); + let (records, root) = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let records = capture.events.json_events(); + let root = capture.spans().into_iter().find(|span| { + span.name == "openshell.http.request" + && span_attribute(span, "openshell.request.id") + .is_some_and(|value| value.as_str() == receipt) + }); + if let Some(root) = root + && records.iter().any(|record| { + record["message"] + .as_str() + .is_some_and(|message| message.starts_with(terminal_prefix)) + && record["metadata"]["correlation_uid"] == receipt + }) + { + break (records, root); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("actual outer terminal record must retain the failed request receipt"); + let correlated = records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == receipt) + .collect::>(); + assert_eq!(correlated.len(), 2, "{records:?}"); + for (index, record) in correlated.iter().enumerate() { + assert_eq!( + record["unmapped"]["request_sequence"], + index + 1, + "{records:?}" + ); + } + let terminal = correlated.last().unwrap(); + assert!( + terminal["message"] + .as_str() + .unwrap() + .starts_with(terminal_prefix) + ); + assert!( + terminal["message"] + .as_str() + .unwrap() + .contains("2 bytes remaining") + ); + assert_eq!(terminal["class_uid"], 4001); + assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); + assert_ne!(receipt, "forged-receipt"); + assert_ne!(root.span_context.trace_id().to_string(), WORKLOAD_TRACE_ID); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_eq!(root.links.links.len(), 1); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + for decision in &correlated[..correlated.len() - 1] { + assert_eq!( + decision["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + } + }; + tokio::time::timeout(std::time::Duration::from_secs(10), scenario) + .await + .expect("real proxy terminal-error exchanges must finish"); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + drop(proxy); + capture.shutdown(); + println!("{COMPLETED}"); +} + +async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) { + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let policy_denied = matches!( + outcome, + Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied + ); + let credential_failure = matches!(outcome, Outcome::UnresolvedCredential); + let secret_resolver = credential_failure.then(|| { + let (_, resolver) = SecretResolver::from_provider_env( + [("OTHER_KEY".into(), "inert-test-value".into())] + .into_iter() + .collect(), + ); + Arc::new(resolver.expect("test credential resolver")) + }); + let mode = if matches!(outcome, Outcome::AuditDenied) { + "audit" + } else { + "enforce" + }; + let path = if policy_denied || matches!(outcome, Outcome::AuditDenied) { + "/blocked" + } else { + "/allowed" + }; + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_api: + name: traced_api + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: rest + enforcement: {mode} + rules: + - allow: {{ method: GET, path: /allowed }} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: vec![executable_path("/workspace/investigate.py")], + }); + let expect_export = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + let subscriber = capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")); + let (mut app, proxy) = tokio::io::duplex(8192); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + let query = if matches!(outcome, Outcome::MalformedQueryPolicyDenied) { + "token=%ff" + } else { + "token=SENTINEL-QUERY" + }; + let target = format!("http://{ip}:{port}{path}?{query}"); + let credential_header = if credential_failure { + "Authorization: Bearer openshell:resolve:env:MISSING\r\n" + } else { + "" + }; + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\n{credential_header}Connection: close\r\n\r\n" + ); + let exchange = async { + tokio::join!( + async { + let result = Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut proxy, + Some(&identity), + Some(( + "192.0.2.20:42123".parse().unwrap(), + "192.0.2.10:3128".parse().unwrap(), + )), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + secret_resolver, + None, + None, + None, + )) + .await; + drop(proxy); + result + }, + async { + if policy_denied || credential_failure { + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(50), + listener.accept() + ) + .await + .is_err(), + "rejected request reached upstream" + ); + } else { + let (mut upstream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + bytes.push(upstream.read_u8().await.unwrap()); + } + assert!( + String::from_utf8_lossy(&bytes) + .starts_with(&format!("GET {path}?token=SENTINEL-QUERY HTTP/1.1\r\n")) + ); + assert!(!String::from_utf8_lossy(&bytes).contains("forged-receipt")); + if !matches!(outcome, Outcome::UpstreamClosed) { + upstream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await.unwrap(); + } + } + }, + async { + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + response + }, + ) + }; + let (result, (), response) = + async { tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)).await } + .with_subscriber(subscriber) + .await + .unwrap(); + result.unwrap(); + let spans = capture.spans(); + let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some(if policy_denied { + 403 + } else if credential_failure { + 500 + } else { + 200 + }); + let response = String::from_utf8(response).unwrap(); + let records = capture.events.json_events(); + let expected_records = if policy_denied || credential_failure { + 1 + } else { + 2 + }; + assert_eq!(records.len(), expected_records, "{records:?}"); + assert!(records.iter().all(|record| record["class_uid"] == 4002)); + let record = &records[0]; + let decision = match outcome { + Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied => "deny", + Outcome::AuditDenied => "audit", + Outcome::Allowed | Outcome::UnresolvedCredential | Outcome::UpstreamClosed => "allow", + }; + assert!( + record["message"] + .as_str() + .unwrap() + .starts_with(&format!("FORWARD_L7 {decision} GET ")) + ); + assert_eq!(record["firewall_rule"]["type"], "l7"); + assert!(record.get("status_id").is_none()); + assert!(record.get("status_detail").is_none()); + for control in &records { + assert_eq!(control["http_request"]["url"]["path"], path); + assert!(control.get("http_response").is_none()); + } + if !policy_denied && !credential_failure { + let forwarded = &records[1]; + assert!( + forwarded["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed GET ") + ); + assert_eq!(forwarded["firewall_rule"]["type"], "opa"); + assert_eq!(forwarded["action_id"], 1); + assert_eq!(forwarded["status_id"], 1); + } + let receipt = record["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_request_record_sequence(&records, receipt); + assert!(!format!("{records:?}").contains("SENTINEL")); + assert!(!format!("{records:?}").contains("forged-receipt")); + assert!(!format!("{records:?}").contains("%ff")); + if let Some(expected) = expected { + assert!( + response.starts_with(&format!("HTTP/1.1 {expected} ")), + "{response}" + ); + assert_eq!(response_request_id(response.as_bytes()), receipt); + if policy_denied { + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["error"], "policy_denied"); + assert!( + body["detail"] + .as_str() + .unwrap() + .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) + ); + assert_eq!(body.as_object().unwrap().len(), 2); + } + } else { + assert!( + response.is_empty(), + "upstream EOF must not synthesize a response" + ); + } + assert_eq!(record["action_id"], if policy_denied { 2 } else { 1 }); + let created = capture.created_span_names(); + assert!( + !created.contains(&"supervisor.egress.deny"), + "inspected paths must never create a deny-only span" + ); + assert_eq!( + created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1 + ); + let roots = spans + .iter() + .filter(|span| span.name == "openshell.http.request") + .collect::>(); + assert_eq!( + roots.len(), + usize::from(expect_export), + "local sampler/export configuration controls request span export" + ); + if roots.is_empty() { + assert!(spans.is_empty()); + assert!(records.iter().all(|record| record.get("trace").is_none())); + } else { + assert_eq!(roots.len(), 1); + let root = roots[0]; + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), WORKLOAD_TRACE_ID); + assert_eq!(root.links.links.len(), 1); + assert_eq!( + root.links.links[0].span_context.trace_id().to_string(), + WORKLOAD_TRACE_ID + ); + assert_eq!( + root.links.links[0].span_context.span_id().to_string(), + WORKLOAD_SPAN_ID + ); + assert!( + root.links.links[0] + .span_context + .trace_state() + .header() + .is_empty() + ); + for record in &records { + assert_eq!( + record["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + if let Some(expected) = expected { + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(expected) + })); + } else { + assert!( + root.attributes + .iter() + .all(|attribute| attribute.key.as_str() != "http.response.status_code") + ); + } + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openshell.request.id" && attribute.value.as_str() == receipt + })); + let stages = spans + .iter() + .filter(|span| span.parent_span_id == root.span_context.span_id()) + .collect::>(); + assert!( + stages + .iter() + .any(|span| span.name == "openshell.policy.evaluate") + ); + assert_eq!( + stages + .iter() + .any(|span| span.name == "openshell.upstream.connect"), + !policy_denied && !credential_failure + ); + assert_eq!( + stages + .iter() + .any(|span| span.name == "openshell.upstream.resolve"), + !policy_denied + ); + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + matches!( + outcome, + Outcome::UpstreamClosed | Outcome::UnresolvedCredential + ), + "a policy refusal is an outcome, not an operational tracing failure" + ); + if credential_failure { + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "credential_injection_failed", + ); + let preparation = stages + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .expect("credential preparation stage"); + assert!(matches!( + preparation.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(preparation, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "credential_resolution_refused", + ); + } + assert!( + stages + .iter() + .all(|span| span.span_context.trace_id() == root.span_context.trace_id()) + ); + assert!(!format!("{spans:?}").contains("SENTINEL")); + } + capture.shutdown(); +} + +#[tokio::test] +async fn inspected_forward_traces_all_outcomes_without_sampling_or_export_dependencies() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + for outcome in [ + Outcome::Allowed, + Outcome::PolicyDenied, + Outcome::AuditDenied, + Outcome::UnresolvedCredential, + Outcome::UpstreamClosed, + ] { + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + forward_case(ip, outcome, sampler).await; + } + } +} + +#[tokio::test] +async fn inspected_forward_malformed_query_preserves_correlated_l7_denial() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + // This malformed query already reaches the L7 deny path. Creating a + // receipt must not add a stricter parse that aborts before the 403 and its + // existing policy decision record. + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + forward_case(ip, Outcome::MalformedQueryPolicyDenied, sampler).await; + } +} + +#[test] +fn inspected_forward_error_receipts_preserve_json_bodies() { + let request_id = "server-owned-receipt"; + let denial = openshell_supervisor_middleware::MiddlewareDenial { + config_name: "content-guard".into(), + reason_code: Some("content_match".into()), + }; + for (original, correlated) in [ + ( + build_json_error_response_with_request_id( + 403, + "Forbidden", + "policy_denied", + "denied by L7 policy", + None, + ), + build_json_error_response_with_request_id( + 403, + "Forbidden", + "policy_denied", + "denied by L7 policy", + Some(request_id), + ), + ), + ( + build_middleware_deny_response("api-policy", &denial, None), + build_middleware_deny_response("api-policy", &denial, Some(request_id)), + ), + ( + build_middleware_failure_response("api-policy", None, None), + build_middleware_failure_response("api-policy", None, Some(request_id)), + ), + ( + build_middleware_unavailable_response("api-policy", None), + build_middleware_unavailable_response("api-policy", Some(request_id)), + ), + ] { + let original = String::from_utf8(original).unwrap(); + let correlated = String::from_utf8(correlated).unwrap(); + let (_, original_body) = original.split_once("\r\n\r\n").unwrap(); + let (headers, correlated_body) = correlated.split_once("\r\n\r\n").unwrap(); + assert_eq!(response_request_id(correlated.as_bytes()), request_id); + assert!(headers.contains(&format!("Content-Length: {}\r\n", correlated_body.len()))); + assert_eq!(correlated_body, original_body); + let body: serde_json::Value = serde_json::from_str(correlated_body).unwrap(); + assert!(body.get("request_id").is_none()); + } +} + +#[tokio::test] +async fn inspected_forward_http_trace_ends_before_websocket_session() { + forward_websocket_trace_case(true).await; +} + +#[tokio::test] +async fn inspected_forward_websocket_frame_denial_stays_outside_http_scope() { + forward_websocket_trace_case(false).await; +} + +async fn forward_websocket_trace_case(allow_frame: bool) { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let frame_rule = if allow_frame { + " - allow: { method: WEBSOCKET_TEXT, path: /ws }" + } else { + "" + }; + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_websocket: + name: traced_websocket + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: websocket + enforcement: enforce + rules: + - allow: {{ method: GET, path: /ws }} +{frame_rule} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let subscriber = capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")); + let (mut app, proxy) = tokio::io::duplex(8192); + let target = format!("http://{ip}:{port}/ws"); + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\n\r\n" + ); + let handler = tokio::spawn( + async move { + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut proxy, + Some(&identity), + Some(( + "192.0.2.20:42123".parse().unwrap(), + "192.0.2.10:3128".parse().unwrap(), + )), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + )) + .await + } + .with_subscriber(subscriber), + ); + let (release, released) = tokio::sync::oneshot::channel(); + let (activity, observed_activity) = tokio::sync::oneshot::channel(); + let (handshake, ready) = tokio::sync::oneshot::channel(); + let upstream = tokio::spawn(async move { + let (mut upstream, _) = listener.accept().await.unwrap(); + let request = read_http_headers(&mut upstream).await; + assert!(String::from_utf8_lossy(&request).starts_with("GET /ws HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n\r\n") + .await + .unwrap(); + handshake.send(()).unwrap(); + let frame = crate::l7::websocket::read_frame_for_test(&mut upstream).await; + if allow_frame { + assert_eq!(frame, masked_text_frame(b"after-http-root")); + upstream.write_all(b"\x81\x02ok").await.unwrap(); + } else { + assert_eq!(masked_close_code(&frame), 1008); + let mut remaining = Vec::new(); + upstream.read_to_end(&mut remaining).await.unwrap(); + assert!( + remaining.is_empty(), + "blocked text must never reach upstream, before or after the policy close" + ); + } + activity.send(std::time::SystemTime::now()).unwrap(); + released.await.unwrap(); + }); + let scenario = async { + ready.await.unwrap(); + let response = read_http_headers(&mut app).await; + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 101 ")); + let receipt = response_request_id(&response); + let root = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let mut roots: Vec<_> = capture + .spans() + .into_iter() + .filter(|span| span.name == "openshell.http.request") + .collect(); + if !roots.is_empty() { + assert_eq!(roots.len(), 1); + break roots.pop().unwrap(); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("HTTP root must finish while the upgraded connection is still open"); + assert!(!handler.is_finished()); + assert!(!upstream.is_finished()); + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(101) + })); + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openshell.request.outcome" + && attribute.value.as_str() == "upgraded" + })); + app.write_all(&masked_text_frame(b"after-http-root")) + .await + .unwrap(); + assert!(root.end_time <= observed_activity.await.unwrap()); + let mut frame = [0; 4]; + app.read_exact(&mut frame).await.unwrap(); + if allow_frame { + assert_eq!(&frame, b"\x81\x02ok"); + assert!(!handler.is_finished()); + } else { + assert_eq!(&frame, b"\x88\x02\x03\xf0"); + } + release.send(()).unwrap(); + drop(app); + upstream.await.unwrap(); + let result = handler.await.unwrap(); + if allow_frame { + result.unwrap(); + } else { + assert!( + result + .expect_err("denied frame must terminate the relay") + .to_string() + .contains("websocket text message denied by policy") + ); + } + let records = capture.events.json_events(); + assert_eq!(records.len(), 4, "{records:?}"); + assert_request_record_sequence(&records[..2], receipt); + for record in &records[..2] { + assert_eq!(record["class_uid"], 4002); + assert_eq!(record["action_id"], 1); + assert_eq!(record["http_request"]["url"]["path"], "/ws"); + assert!(record.get("http_response").is_none()); + assert_eq!( + record["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("FORWARD_L7 allow GET ") + ); + assert_eq!(records[0]["firewall_rule"]["type"], "l7"); + assert!(records[0].get("status_id").is_none()); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed GET ") + ); + assert_eq!(records[1]["firewall_rule"]["type"], "opa"); + assert_eq!(records[1]["status_id"], 1); + assert_eq!(records[2]["class_uid"], 4001); + assert_eq!(records[2]["activity_name"], "Upgrade"); + assert!( + records[2]["message"] + .as_str() + .unwrap() + .starts_with("101 Switching Protocols — websocket parsed relay ") + ); + assert!(records[2].get("http_response").is_none()); + assert!(records[2].get("http_request").is_none()); + assert_eq!(records[2]["metadata"]["correlation_uid"], receipt); + assert_eq!(records[2]["unmapped"]["request_sequence"], 3); + assert_eq!(records[2]["unmapped"]["http_event_kind"], "connection"); + // NetworkActivity has no trace field; the receipt ties the unchanged + // 101 record to the completed HTTP root without changing its schema. + assert!(records[2].get("trace").is_none()); + assert_eq!( + span_attribute(&root, "openshell.request.id") + .unwrap() + .as_str(), + receipt + ); + assert_eq!(records[3]["class_uid"], 4001); + assert!( + records[3]["message"] + .as_str() + .unwrap() + .starts_with(if allow_frame { + "WEBSOCKET_L7_REQUEST allow WEBSOCKET_TEXT " + } else { + "WEBSOCKET_L7_REQUEST deny WEBSOCKET_TEXT " + }) + ); + assert_eq!(records[3]["firewall_rule"]["type"], "l7-websocket"); + assert_eq!(records[3]["action_id"], if allow_frame { 1 } else { 2 }); + for record in &records[3..] { + assert!(record["metadata"].get("correlation_uid").is_none()); + assert!(record["unmapped"].get("request_sequence").is_none()); + assert!(record["unmapped"].get("http_event_kind").is_none()); + assert!(record.get("trace").is_none()); + } + let created = capture.created_span_names(); + assert_eq!( + created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1, + "WebSocket frames must not create HTTP request spans" + ); + assert!( + !created.contains(&"supervisor.egress.deny"), + "WebSocket NetworkActivity decisions must not gain HTTP denial spans" + ); + assert!(!format!("{records:?}").contains("after-http-root")); + }; + tokio::time::timeout(std::time::Duration::from_secs(5), scenario) + .await + .unwrap(); + capture.shutdown(); +} diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index 0d26319749..8c85b1e380 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -80,6 +80,7 @@ async fn forward_inspection_selects_only_the_owner_admitting_the_request() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare admitted forward request"); @@ -130,6 +131,7 @@ async fn forward_inspection_retains_installation_through_guarded_write() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare authenticated request"); @@ -178,6 +180,7 @@ async fn forward_inspection_retains_installation_through_guarded_write() { response_middleware: None, endpoint_observer: None, }, + None, ); let result = tokio::time::timeout(std::time::Duration::from_secs(5), relay) .await diff --git a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs index 1602913885..d6871ca941 100644 --- a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs +++ b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs @@ -16,6 +16,7 @@ async fn paired_credential_denial_shares_one_deny_span() { 443, "credential-policy", &tracing::Span::none(), + None, ); }) .await; @@ -38,6 +39,7 @@ async fn repeated_credential_denials_keep_decision_spans_separate() { 443, "credential-policy", &connect, + None, ); } let allowed = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -133,6 +135,7 @@ async fn forward_denials_use_the_saved_connect_parent() { &decision, None, None, + None, )) .unwrap(); assert!(output.starts_with(b"HTTP/1.1 403 Forbidden")); @@ -194,7 +197,7 @@ async fn forward_evaluation_error_keeps_the_http_decision_policy_context() { "evaluation unavailable", ); // Use the same grouping as the forward-proxy production path. - emit_forward_l7_decision(decision, Some(error), &connect); + emit_forward_l7_decision(decision, Some(error), &connect, None); }) .await; diff --git a/crates/openshell-supervisor-network/src/telemetry.rs b/crates/openshell-supervisor-network/src/telemetry.rs index 73e573b985..8de4530dac 100644 --- a/crates/openshell-supervisor-network/src/telemetry.rs +++ b/crates/openshell-supervisor-network/src/telemetry.rs @@ -1,13 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Keep a deny span active during OCSF emission. +//! Emit egress audit records in their owning trace context. use std::collections::HashMap; use std::sync::{LazyLock, Mutex}; use std::time::{Duration, Instant}; use openshell_ocsf::{ActionId, HttpActivityEvent, OcsfEvent}; +use tracing_opentelemetry::OpenTelemetrySpanExt; /// Emit one egress event, entering an INFO deny span if it is denied HTTP Activity. macro_rules! egress_emit { @@ -38,8 +39,17 @@ fn as_http(event: &OcsfEvent) -> Option<&HttpActivityEvent> { /// a span. The span describes the denied HTTP record, or the first HTTP record /// when only a companion finding is denied. A saved `parent` keeps the span in /// the connect trace after authorization exits. +/// +/// An inspected request already owns an all-outcome span. Its ambient request +/// scope suppresses denial-only spans even for legacy emitters and when tracing +/// is disabled, filtered, or unsampled. #[allow(clippy::disallowed_macros)] pub fn emit_decision(events: impl IntoIterator, parent: Option<&tracing::Span>) { + if crate::l7::correlation::is_inspected_request() { + let request_span = crate::l7::correlation::current_request_span(); + emit_inspected_decision(events, request_span.as_ref()); + return; + } let events: Vec<_> = events.into_iter().collect(); let span = if events.iter().any(is_denied) { events @@ -63,6 +73,27 @@ pub fn emit_decision(events: impl IntoIterator, parent: Option } } +/// Inspected requests own an all-outcome span, so their records never create a +/// denial-only span, including when that request span is disabled or unsampled. +#[allow(clippy::disallowed_macros)] +pub fn emit_inspected_decision( + events: impl IntoIterator, + parent: Option<&tracing::Span>, +) { + let emit = || { + for event in events { + openshell_ocsf::ocsf_emit!(event); + } + }; + let context = parent.map_or_else(opentelemetry::Context::new, OpenTelemetrySpanExt::context); + let _context = context.attach(); + if let Some(parent) = parent { + parent.in_scope(emit); + } else { + emit(); + } +} + /// Returns `None` when deny spans are disabled or the destination has used up /// its deny spans. Disabled spans spend no tokens. fn deny_span(event: &HttpActivityEvent, parent: Option<&tracing::Span>) -> Option { diff --git a/crates/openshell-supervisor-network/src/telemetry/tests.rs b/crates/openshell-supervisor-network/src/telemetry/tests.rs index 7a3388ad0d..59d9f2e408 100644 --- a/crates/openshell-supervisor-network/src/telemetry/tests.rs +++ b/crates/openshell-supervisor-network/src/telemetry/tests.rs @@ -81,6 +81,10 @@ static REGISTER: LazyLock<()> = LazyLock::new(|| { )); }); +pub fn init_trace_extractor() { + LazyLock::force(®ISTER); +} + fn capture( log_level: &str, provider: &opentelemetry_sdk::trace::SdkTracerProvider, @@ -600,3 +604,85 @@ async fn disabled_deny_spans_spend_no_tokens() { assert!(super::LIMITER.lock().unwrap().buckets.is_empty()); assert!(super::LIMITER.lock().unwrap().global.is_none()); } + +#[tokio::test] +async fn inspected_audit_records_do_not_depend_on_sampling_or_create_deny_spans() { + let _lock = tracing_test_lock().await; + LazyLock::force(®ISTER); + for sampled in [false, true] { + super::reset_deny_span_limiter(); + let exporter = opentelemetry_sdk::trace::InMemorySpanExporter::default(); + let sampler = if sampled { + opentelemetry_sdk::trace::Sampler::AlwaysOn + } else { + opentelemetry_sdk::trace::Sampler::AlwaysOff + }; + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_sampler(sampler) + .with_simple_exporter(exporter.clone()) + .build(); + let request_id = uuid::Uuid::new_v4().to_string(); + let events = capture("warn", &provider, || { + let request = tracing::info_span!(parent: None, "openshell.http.request"); + let mut denied = http_deny("blocked.example", "middleware_blocked"); + let openshell_ocsf::OcsfEvent::HttpActivity(event) = &mut denied else { + unreachable!("http_deny returns an HTTP activity"); + }; + event.base.metadata.correlation_uid = Some(request_id.clone()); + super::emit_inspected_decision([denied], Some(&request)); + }); + provider.force_flush().unwrap(); + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), usize::from(sampled)); + assert!( + spans + .iter() + .all(|span| span.name == "openshell.http.request") + ); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(events[0]["action_id"], 2); + if sampled { + assert_eq!( + events[0]["trace"]["uid"], + spans[0].span_context.trace_id().to_string() + ); + } else { + assert!(events[0].get("trace").is_none()); + } + assert!(super::LIMITER.lock().unwrap().buckets.is_empty()); + assert!(super::LIMITER.lock().unwrap().global.is_none()); + provider.shutdown().unwrap(); + } +} + +#[tokio::test] +async fn inspected_emission_without_a_request_span_never_falls_back_to_deny_tracing() { + let (events, received) = exported("warn", || { + use opentelemetry::trace::TraceContextExt as _; + let ambient = opentelemetry::trace::SpanContext::new( + opentelemetry::trace::TraceId::from(1), + opentelemetry::trace::SpanId::from(2), + opentelemetry::trace::TraceFlags::SAMPLED, + true, + opentelemetry::trace::TraceState::default(), + ); + let _ambient = opentelemetry::Context::new() + .with_remote_span_context(ambient.clone()) + .attach(); + super::emit_inspected_decision([http_deny("blocked.example", "no_parent")], None); + super::emit_inspected_decision( + [http_deny("blocked.example", "disabled_parent")], + Some(&tracing::Span::none()), + ); + assert_eq!( + opentelemetry::Context::current().span().span_context(), + &ambient + ); + }) + .await; + assert_eq!(events.len(), 2); + assert!(received.spans.is_empty()); + assert!(events.iter().all(|event| event["action_id"] == 2)); + assert_no_deny(&events, &received); +} diff --git a/crates/openshell-supervisor-network/src/test_support.rs b/crates/openshell-supervisor-network/src/test_support.rs index 5c302b91de..4ad244dae5 100644 --- a/crates/openshell-supervisor-network/src/test_support.rs +++ b/crates/openshell-supervisor-network/src/test_support.rs @@ -1,7 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Credential, executable identity, and socket-owner subprocess fixtures. +//! Credential, identity, socket-owner and request-tracing fixtures. + +pub mod request_tracing; + +pub fn response_request_id(response: &[u8]) -> &str { + let end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("complete response headers"); + std::str::from_utf8(&response[..end]) + .expect("ASCII response headers") + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("response carries a request receipt") +} pub fn header_placement_metadata( keys: &[&str], diff --git a/crates/openshell-supervisor-network/src/test_support/request_tracing.rs b/crates/openshell-supervisor-network/src/test_support/request_tracing.rs new file mode 100644 index 0000000000..2cee89d2ab --- /dev/null +++ b/crates/openshell-supervisor-network/src/test_support/request_tracing.rs @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, Mutex}; + +use openshell_ocsf::test_support::OcsfCapture; +use opentelemetry_sdk::trace::{InMemorySpanExporter, Sampler, SdkTracerProvider, SpanData}; +use tracing_subscriber::{layer::SubscriberExt as _, registry::LookupSpan}; + +pub fn span_attribute<'a>(span: &'a SpanData, name: &str) -> Option<&'a opentelemetry::Value> { + span.attributes + .iter() + .rev() + .find(|attribute| attribute.key.as_str() == name) + .map(|attribute| &attribute.value) +} + +#[derive(Clone, Default)] +struct CreatedSpans(Arc>>); + +impl tracing_subscriber::Layer for CreatedSpans { + fn on_new_span( + &self, + attributes: &tracing::span::Attributes<'_>, + _id: &tracing::Id, + _context: tracing_subscriber::layer::Context<'_, S>, + ) { + self.0.lock().unwrap().push(attributes.metadata().name()); + } +} + +pub struct RequestTracing { + pub events: OcsfCapture, + exporter: InMemorySpanExporter, + provider: Option, + created: CreatedSpans, +} + +impl RequestTracing { + pub fn new(sampler: Option) -> Self { + crate::telemetry::tests::init_trace_extractor(); + let exporter = InMemorySpanExporter::default(); + let provider = sampler.map(|sampler| { + SdkTracerProvider::builder() + .with_sampler(sampler) + .with_simple_exporter(exporter.clone()) + .build() + }); + Self { + events: OcsfCapture::default(), + exporter, + provider, + created: CreatedSpans::default(), + } + } + + pub fn subscriber( + &self, + ) -> impl tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync + 'static { + let layer = self + .provider + .as_ref() + .map(|provider| openshell_otel::layer(provider, "request-test")); + tracing_subscriber::registry() + .with(self.events.clone()) + .with(self.created.clone()) + .with(layer) + } + + pub fn spans(&self) -> Vec { + if let Some(provider) = &self.provider { + provider.force_flush().unwrap(); + } + self.exporter.get_finished_spans().unwrap() + } + + pub fn created_span_names(&self) -> Vec<&'static str> { + self.created.0.lock().unwrap().clone() + } + + pub fn shutdown(&self) { + if let Some(provider) = &self.provider { + provider.shutdown().unwrap(); + } + } +} diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 2f4b52c10f..42ab054bd1 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -311,12 +311,94 @@ HTTP Activity and API Activity records emitted inside a sampled OpenTelemetry sp OTLP log export links every OCSF record, whatever its class, to the active sampled span through the log record's own trace and span ID fields. Use those fields for log-to-trace links in your backend. -With OTLP trace export enabled on a dedicated supervisor, denied HTTP requests can create an INFO `supervisor.egress.deny` span around the audit records for that decision. The span records the policy, reason, and destination when available. At the default log level, every deny span is a root span. At debug level, FORWARD policy, destination, credential-binding, and request middleware denials can join the available connect trace and its authorization, resolve, and dial spans. Other deny spans, including L7 denials inside CONNECT tunnels and response middleware denials, remain root spans. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. Workload `traceparent` headers never supply supervisor correlation. +With OTLP trace export enabled on a dedicated supervisor, each inspected HTTP request has an INFO `openshell.http.request` span for its allowed, denied or failed outcome. Children measure the policy, middleware, credential, upstream and response stages that actually run. HTTP upgrade tracing ends before the upgraded WebSocket session. -To bound telemetry volume when a workload retries blocked destinations, the supervisor creates at most 10 deny spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. Records can retain correlation to an already active sampled span, including a connect span at debug level. +The request span ends after its owned stages, not when response headers are logged. `openshell.upstream.send` measures request writes and flush, not the full upstream round trip. Response handling has a separate span. + +The supervisor owns the request trace. A valid workload `traceparent`, sampled or unsampled, becomes an explicitly untrusted link, never its parent. Invalid or duplicate context is ignored; workload `tracestate` is not adopted. Your backend may show the agent and supervisor as separate linked traces rather than one nested tree. + +Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Non-inspected HTTP denials use that span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records have no supervisor trace IDs. Request ID correlation works independently of those exporters. + +The non-inspected denial-span limiter allows at most 10 spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. These denial-specific budgets do not select inspected request spans, which cover every outcome rather than denials alone. Gateway records and their OTLP trace context can inherit the caller's incoming `traceparent`, including before authentication succeeds. The caller also decides whether they are correlated at all, because an unsampled `traceparent` suppresses `trace.uid`. Treat gateway trace IDs as correlation hints, not evidence of the caller's identity, and do not treat a missing `trace.uid` as meaningful. +## Join Inspected HTTP Requests + +The supervisor assigns each inspected HTTP request a UUID shared by middleware +`RequestContext.request_id`, OCSF `metadata.correlation_uid`, and the +`x-openshell-request-id` response header. It also exports the ID as the OTLP log +attribute `ocsf.metadata.correlation_uid`. + +This header is reserved for OpenShell: workload and upstream values are removed, +and middleware cannot set or remove it. Use a different header for +application-owned correlation. The receipt is a join key, not proof of identity, +authorization or downstream execution. L4 and uninspected traffic do not provide +this receipt guarantee. OpenShell does not add the receipt to JSON error bodies. + +All OCSF event classes emitted in the inspected request scope carry the request +ID, including Detection Findings. Each control emits its own records with event +IDs, timestamps, classification and decision fields. Use +`unmapped.http_event_kind` to select the record kind: + +| Value | Record kind | +|---|---| +| `decision` | HTTP control decision. | +| `middleware_invocation` | Request, response or WebSocket middleware invocation. | +| `token_grant` | Provider token grant. | +| `finding` | Detection Finding. | +| `connection` | Network or SSH activity, including the HTTP upgrade handoff. | +| `lifecycle` | Process or application lifecycle. | +| `configuration` | Configuration state change. | +| `authentication` | Authentication activity. | +| `event` | Other event classes. | + +Middleware invocation records use `unmapped.middleware_phase` with values +`request`, `response`, `websocket_preflight`, `websocket_session_start` or +`websocket_message`. This field is separate from `unmapped.phase`, which describes +a finding's phase, such as `pre_return`. Attribution and transformation details +are recorded on the individual invocation. + +`unmapped.request_sequence` starts at 1 for each request and records local +emission order. It does not guarantee downstream delivery order or complete +collection. To count policy outcomes, group by `metadata.correlation_uid` and +select the actual policy or firewall control record. Do not count every HTTP +Activity or `decision` record as another request. + +Errors that end an inspected request retain that request's ID. A policy reload +records the active request before closing it; an idle connection has no request +ID. The `101` upgrade handoff record shares the HTTP handshake ID. Later +WebSocket frame events do not reuse that ID. + +The request ID join and enforcement work without tracing, when spans are not +sampled, and when the collector is unavailable. Sampled OCSF trace fields identify +the supervisor trace; the request ID remains the join key when those fields are +absent. An instrumented client can record the receipt on its own trace and use it +to find OpenShell's security records; it does not need to receive those records +inside the sandbox. + +For inspected-request traces, query span name `openshell.http.request` +and join through `openshell.request.id`. A refusal +can have outcome `"denied"` or `"failed"`; use the original OCSF control record +for its policy and reason. For non-inspected denials, query `supervisor.egress.deny`. + +### Size Supervisor Trace Collection + +Denial-span budgets do not bound inspected request spans, which include +allowed traffic at INFO level. At `R` inspected requests per second and an +average of `S` descendant spans per request, expect approximately +`R × (1 + S)` spans per second before exporter or collector loss, plus background +and lifecycle spans. Size the collector for this traffic before enabling export. + +The bounded exporter queue does not cap span generation. Collector-side sampling +reduces stored traces, not the supervisor's span-generation cost. The gateway's +`OTEL_TRACES_SAMPLER` setting does not configure supervisor sampling; operator +control is tracked in +[#5362](https://github.com/NVIDIA/OpenShell/issues/5362). + +Leave supervisor trace export disabled if your deployment requires a +denial-only volume bound. OCSF records and request receipts still work without it. + ## OCSF Event Classes in JSON The `class_uid` field identifies the event type: diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index 3cbdba70a3..abe075e27b 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -28,11 +28,11 @@ In the Helm chart, set the table under `gatewayConfig`. The gateway passes the e | Gateway requests | One server span for each inbound gRPC or HTTP request. Request spans carry `method`, `path`, and the `request_id` that also appears in gateway logs. | | Gateway background work | Reconciliation, driver watches, and credential refresh. These start their own traces because no request supplies a parent. Credential refresh creates a span only when a credential is due or needs cleanup. | | Compute drivers | Child spans for each compute-driver operation the gateway performs, from both the gateway's side and the driver's side. | -| Sandbox supervisors | Supervisor startup, such as loading policy. | +| Sandbox supervisors | Supervisor startup, such as loading policy, plus inspected HTTP requests and the stages they execute. | These are OpenShell's own spans. Spans that the agent produces travel separately, through the [agent trace relay](/observability/overview#agent-traces). -Spans for routine or high-volume operations use DEBUG level, and the default INFO level does not export them. These include store operations, health checks, the RPCs that supervisors call on a timer (`GetSandboxConfig` and `ReportProviderReadiness`), and how supervisors handle egress connections. Set the gateway or sandbox log level to `debug` to include them. +Store operations, health checks, periodic supervisor RPCs (`GetSandboxConfig` and `ReportProviderReadiness`), and connection-level egress diagnostics use DEBUG level. Set the gateway or sandbox log level to `debug` to include them. Inspected HTTP request spans and their executed stage children use INFO level, including allowed requests. Their volume is not limited by the legacy denial-span budgets. See [query and capacity migration](/observability/ocsf-json-export#update-existing-queries) before enabling supervisor trace export. ## Span Names @@ -43,7 +43,7 @@ Each component names its spans after the operation it performs: | Gateway requests | The gRPC service and method, or the HTTP method for other requests | `openshell.v1.OpenShell/CreateSandbox`, `GET` | | Gateway background work | The task | `reconcile.sandboxes`, `refresh.provider_credentials` | | Compute drivers | The compute-driver RPC, with child spans prefixed by the driver name | `openshell.compute.v1.ComputeDriver/CreateSandbox`, `docker.provision` | -| Sandbox supervisors | Prefixed with `supervisor.` | `supervisor.startup`, `supervisor.policy.load` | +| Sandbox supervisors | Lifecycle spans use `supervisor.`; inspected HTTP uses request and stage operation names | `supervisor.startup`, `supervisor.policy.load`, `openshell.http.request`, `openshell.policy.evaluate` | The gateway's client span and the driver's server span for a compute-driver call share the same name, which also appears in `rpc.method`. Their service names and span kinds tell them apart. A driver watch stream records a terminal status when it observes one. If the gateway stops consuming the stream first, the span status stays unset.