diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index dd8af76169..e5e4647474 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -94,3 +94,8 @@ filtering preserves raw HTTP bytes and borrows unchanged header blocks. It removes only protected receipt fields and their `Connection` or `Trailer` nominations, including folded continuations, without adding general response-header validation. + +Credential preparation stages record bounded mechanism, placement and preparation +outcomes, never credential values. A configured resolver without placeholders is +`not_needed`; `prepared` means resolution or signing succeeded, not that the +upstream received the request. Transport failures belong to their own stages. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index 1d4cfe36fa..ffa868cb97 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -83,6 +83,17 @@ pub fn record_current_failure(outcome: HttpOutcome, reason: &'static str) { } } +pub fn record_credential_preparation( + span: &tracing::Span, + mechanism: &'static str, + placement: &'static str, + preparation: &'static str, +) { + span.record("openshell.credential.mechanism", mechanism); + span.record("openshell.credential.placement", placement); + span.record("openshell.credential.preparation", preparation); +} + fn enrich_request(event: &mut openshell_ocsf::BaseEventData) { let Some(scope) = current_request().filter(|scope| scope.inspected) else { return; diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index 4128083134..6ec0385d0a 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -514,12 +514,22 @@ network_policies: #[tokio::test] async fn inspected_allow_creates_only_the_stages_that_run() { - for sampled in [true, false] { + for (sampled, credential) in [(true, false), (false, false), (true, true)] { + let mut fixture = rest_fixture("POST", false); + let authorization = if credential { + let (_, resolver) = openshell_core::secrets::SecretResolver::from_provider_env( + std::iter::once(("PRESENT".into(), "SENTINEL-CREDENTIAL".into())).collect(), + ); + fixture.ctx.secret_resolver = resolver.map(Arc::new); + "Authorization: Basic dXNlcjpvcGVuc2hlbGw6cmVzb2x2ZTplbnY6UFJFU0VOVA==\r\n" + } else { + "" + }; let evidence = capture_request_spans( sampled, Box::pin(run_exchange( - rest_fixture("POST", false), - request("rest.example.test", "/item", "", "{}"), + fixture, + request("rest.example.test", "/item", authorization, "{}"), Some(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), )), ) @@ -543,10 +553,30 @@ async fn inspected_allow_creates_only_the_stages_that_run() { ("openshell.http.response", true), ("openshell.middleware.request", false), ("openshell.middleware.response", false), - ("openshell.credential.prepare", false), + ("openshell.credential.prepare", credential), ] { assert_eq!(names.contains(&name), expected, "{name}: {names:?}"); } + if credential { + let preparation = evidence + .spans + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .unwrap(); + assert_eq!( + span_attribute(preparation, "openshell.credential.preparation") + .unwrap() + .as_str(), + "prepared" + ); + assert!( + String::from_utf8_lossy(&evidence.output.forwarded) + .contains("Basic dXNlcjpTRU5USU5FTC1DUkVERU5USUFM") + ); + let metadata = format!("{:?}", evidence.spans); + assert!(!metadata.contains("SENTINEL-CREDENTIAL")); + assert!(!metadata.contains("dXNlcjpTRU5USU5FTC1DUkVERU5USUFM")); + } } let records = assert_control_records(&evidence, &[("l7", "Allowed")], &[], "/item"); assert!(records[0].get("status").is_none()); diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index d53b1e0abd..341bbd9e6c 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -998,23 +998,35 @@ async fn failure_then_reconnect(route_selection: bool) { } #[tokio::test] -async fn token_grant_refusal_never_creates_a_denial_only_span() { +async fn token_grant_preparation_reports_success_and_refusal_without_denial_only_spans() { use super::correlation_dispatch_tests::capture_request_spans; + use crate::test_support::request_tracing::span_attribute; - for sampled in [true, false] { + for (sampled, refuses) in [(true, true), (false, true), (true, false)] { let evidence = capture_request_spans(sampled, async { let fixture = Fixture::new(&[route("a", "/a/**")], &[("/a/**", "a", "aud-a")], 1); - fixture.resolver.fail_once.store(true, Ordering::SeqCst); + fixture.resolver.fail_once.store(refuses, Ordering::SeqCst); let mut connection = fixture.connect(); let response = connection.exchange("GET", "/a/item", true).await; - assert!(connection.observes_eof().await); + if refuses { + assert!(connection.observes_eof().await); + } let capture = connection.finish().await; - assert_eq!(capture.bytes_seen, 0); + if refuses { + assert_eq!(capture.bytes_seen, 0); + } else { + assert_eq!(capture.requests.len(), 1); + assert_authorization(&capture.requests[0], "aud-a"); + } response }) .await; evidence.assert_no_deny_span(sampled); - assert!(evidence.output.starts_with("HTTP/1.1 502")); + assert!(evidence.output.starts_with(if refuses { + "HTTP/1.1 502" + } else { + "HTTP/1.1 204" + })); if sampled { assert_eq!( evidence @@ -1024,12 +1036,42 @@ async fn token_grant_refusal_never_creates_a_denial_only_span() { .count(), 1 ); - assert!( + let preparation = evidence + .spans + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .unwrap(); + for (field, expected) in [ + ("openshell.credential.mechanism", "token_grant"), + ("openshell.credential.placement", "header"), + ( + "openshell.credential.preparation", + if refuses { "refused" } else { "prepared" }, + ), + ] { + assert_eq!( + span_attribute(preparation, field).unwrap().as_str(), + expected + ); + } + assert_eq!( + span_attribute(preparation, "openshell.credential.failure_code") + .map(|value| value.as_str()), + refuses.then_some(std::borrow::Cow::Borrowed("unavailable")), + ); + assert_eq!( + span_attribute(preparation, "openshell.credential.failure_stage") + .map(|value| value.as_str()), + refuses.then_some(std::borrow::Cow::Borrowed("token_endpoint")), + ); + assert_eq!( evidence .spans .iter() - .all(|span| span.name != "openshell.upstream.send") + .any(|span| span.name == "openshell.upstream.send"), + !refuses, ); + assert!(!format!("{:?}", evidence.spans).contains("inert-aud-a")); } let records: Vec<_> = evidence .events @@ -1044,12 +1086,19 @@ async fn token_grant_refusal_never_creates_a_denial_only_span() { .unwrap() .starts_with("L7_REQUEST allow ") ); - assert_eq!(records[1]["action"], "Denied"); + assert_eq!( + records[1]["action"], + if refuses { "Denied" } else { "Allowed" } + ); assert!( records[1]["message"] .as_str() .unwrap() - .starts_with("Token grant failed ") + .starts_with(if refuses { + "Token grant failed " + } else { + "Token grant successful " + }) ); for record in records { assert!(record.get("http_response").is_none()); diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs index 1d9c36a509..c0a48a5ea9 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -95,6 +95,20 @@ impl RequestTrace { if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() { return Span::none(); } + if matches!(stage, RequestStage::CredentialPrepare) { + return tracing::info_span!( + parent: &self.span, + "openshell.http.stage", + otel.name = stage.name(), + otel.status_code = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + openshell.credential.mechanism = tracing::field::Empty, + openshell.credential.placement = tracing::field::Empty, + openshell.credential.preparation = tracing::field::Empty, + openshell.credential.failure_code = tracing::field::Empty, + openshell.credential.failure_stage = tracing::field::Empty, + ); + } tracing::info_span!( parent: &self.span, "openshell.http.stage", diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index caec858eaf..1cdc7b29ac 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,7 +27,9 @@ pub(crate) use request_body::{ }; use crate::l7::EndpointObserver; -use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; +use crate::l7::correlation::{ + HttpOutcome, RequestCorrelation, RequestStage, record_credential_preparation, +}; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -1079,16 +1081,26 @@ where offered_subprotocols: request.subprotocols.clone(), }); - let rewrite_span = - if options.resolver.is_some() || contains_reserved_credential_marker_bytes(&header_bytes) { - correlation.map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::CredentialPrepare) - }) - } else { - tracing::Span::none() - }; + let has_credential_marker = contains_reserved_credential_marker_bytes(&header_bytes); + let rewrite_span = if options.resolver.is_some() || has_credential_marker { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }) + } else { + tracing::Span::none() + }; let rewrite_result = rewrite_span.in_scope(|| rewrite_http_header_block(&header_bytes, options.resolver)); + record_credential_preparation( + &rewrite_span, + "static", + "request_head", + match &rewrite_result { + Ok(result) if result.rewritten != header_bytes => "prepared", + Ok(_) => "not_needed", + Err(_) => "refused", + }, + ); rewrite_span.record( "otel.status_code", if rewrite_result.is_ok() { @@ -1260,7 +1272,7 @@ where } let sign = || { - crate::sigv4::apply_sigv4_to_request( + let result = crate::sigv4::apply_sigv4_to_request( &full_request, options.host, ®ion, @@ -1268,7 +1280,14 @@ where access_key, secret_key, session_token, - ) + ); + record_credential_preparation( + &tracing::Span::current(), + "sigv4", + "header", + if result.is_ok() { "prepared" } else { "failed" }, + ); + result }; let signed = correlation.map_or_else(sign, |correlation| { correlation.run_stage_sync( @@ -1289,7 +1308,7 @@ where } _ => SignableBody::UnsignedPayload, }; - crate::sigv4::apply_sigv4_headers_only_with_body( + let result = crate::sigv4::apply_sigv4_headers_only_with_body( &rewrite_result.rewritten, options.host, ®ion, @@ -1298,7 +1317,14 @@ where secret_key, session_token, signable_body, - ) + ); + record_credential_preparation( + &tracing::Span::current(), + "sigv4", + "header", + if result.is_ok() { "prepared" } else { "failed" }, + ); + result }; let signed_headers = correlation.map_or_else(sign, |correlation| { correlation.run_stage_sync( @@ -1383,16 +1409,39 @@ where ))); } } else if options.request_body_credential_rewrite { - let body = collect_and_rewrite_request_body( - req, - client, - &rewrite_result.rewritten, - header_str, - &req.raw_header[header_end..], - options.resolver, - options.generation_guard, - ) - .await?; + let prepare = async { + let result = collect_and_rewrite_request_body( + req, + client, + &rewrite_result.rewritten, + header_str, + &req.raw_header[header_end..], + options.resolver, + options.generation_guard, + ) + .await; + if result.is_err() { + record_credential_preparation( + &tracing::Span::current(), + "static", + "body", + "failed", + ); + } + result + }; + let body = if let Some(correlation) = correlation { + correlation + .run_stage( + RequestStage::CredentialPrepare, + HttpOutcome::PlatformError, + "credential_preparation_failed", + Box::pin(prepare), + ) + .await + } else { + prepare.await + }?; ensure_credential_generation_current(options)?; upstream.write_all(&body.headers).await.into_diagnostic()?; if !body.body.is_empty() { @@ -2030,6 +2079,12 @@ async fn collect_and_rewrite_request_body( "request body credential rewrite cannot resolve placeholders without explicit body framing" )); } + record_credential_preparation( + &tracing::Span::current(), + "static", + "body", + "not_needed", + ); Ok(PreparedRequestBody { headers: rewritten_headers.to_vec(), body: already_read.to_vec(), @@ -2092,6 +2147,7 @@ fn rewrite_buffered_body( resolver: Option<&SecretResolver>, ) -> Result<(Vec, Vec)> { if body.is_empty() { + record_credential_preparation(&tracing::Span::current(), "static", "body", "not_needed"); return Ok((headers.to_vec(), body)); } @@ -2102,12 +2158,14 @@ fn rewrite_buffered_body( "request body credential rewrite found placeholders in an unsupported content type" )); } + record_credential_preparation(&tracing::Span::current(), "static", "body", "not_needed"); return Ok((headers.to_vec(), body)); } let mut text = String::from_utf8(body) .map_err(|_| miette!("request body credential rewrite requires UTF-8 text bodies"))?; if !contains_reserved_credential_marker(&text) { + record_credential_preparation(&tracing::Span::current(), "static", "body", "not_needed"); return Ok((headers.to_vec(), text.into_bytes())); } @@ -2134,6 +2192,7 @@ fn rewrite_buffered_body( let body = text.into_bytes(); let headers = set_content_length(headers, body.len())?; + record_credential_preparation(&tracing::Span::current(), "static", "body", "prepared"); Ok((headers, body)) } @@ -11919,8 +11978,13 @@ mod tests { crate::l7::CredentialSigning::SigV4Body, crate::l7::CredentialSigning::SigV4NoBody, ] { - for host in ["s3.us-east-1.amazonaws.com", "invalid host"] { - let succeeds = host != "invalid host"; + for (host, upstream_open) in [ + ("s3.us-east-1.amazonaws.com", true), + ("invalid host", true), + ("s3.us-east-1.amazonaws.com", false), + ] { + let signing_succeeds = host != "invalid host"; + let succeeds = signing_succeeds && upstream_open; let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); @@ -11928,6 +11992,10 @@ mod tests { .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") .await .unwrap(); + let mut upstream_side = Some(upstream_side); + if !upstream_open { + drop(upstream_side.take()); + } let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { let ctx = crate::l7::relay::L7EvalContext { host: host.into(), @@ -11960,7 +12028,9 @@ mod tests { assert_eq!(result.is_ok(), succeeds, "{result:?}"); drop(proxy_to_upstream); let mut forwarded = String::new(); - upstream_side.read_to_string(&mut forwarded).await.unwrap(); + if let Some(mut upstream_side) = upstream_side { + upstream_side.read_to_string(&mut forwarded).await.unwrap(); + } if succeeds { assert!(forwarded.contains("authorization:"), "{forwarded}"); } else { @@ -11976,19 +12046,43 @@ mod tests { .collect::>(); assert_eq!(stages.len(), 2, "header resolution and signing"); assert_eq!(stages[0].status, opentelemetry::trace::Status::Ok); + assert_eq!( + span_attribute(stages[0], "openshell.credential.preparation") + .unwrap() + .as_str(), + "not_needed", + "a configured resolver without placeholders does not prepare a credential", + ); let signing = stages[1]; + for (field, expected) in [ + ("openshell.credential.mechanism", "sigv4"), + ("openshell.credential.placement", "header"), + ( + "openshell.credential.preparation", + if signing_succeeds { + "prepared" + } else { + "failed" + }, + ), + ] { + assert_eq!(span_attribute(signing, field).unwrap().as_str(), expected); + } assert_eq!( matches!(signing.status, opentelemetry::trace::Status::Error { .. }), - !succeeds, + !signing_succeeds, ); assert_eq!( span_attribute(signing, "openshell.request.failure_reason") .map(|value| value.as_str()), - (!succeeds).then_some(Cow::Borrowed("credential_signing_failed")), + (!signing_succeeds).then_some(Cow::Borrowed("credential_signing_failed")), ); - if succeeds { + if signing_succeeds { assert_eq!(signing.status, opentelemetry::trace::Status::Ok); } + let metadata = format!("{spans:?}"); + assert!(!metadata.contains("AKIATESTKEY")); + assert!(!metadata.contains("test-secret-key")); capture.shutdown(); } } diff --git a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs index eb0069532c..f90d977a3f 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs @@ -866,9 +866,8 @@ where head = require_identity_response_coding(&head) .map_err(LiveWriteError::Failed)?; } - let credential_span = if options.resolver.is_some() - || contains_reserved_credential_marker_bytes(&head) - { + let has_credential_marker = contains_reserved_credential_marker_bytes(&head); + let credential_span = if options.resolver.is_some() || has_credential_marker { correlation.map_or_else(tracing::Span::none, |correlation| { correlation.stage(RequestStage::CredentialPrepare) }) @@ -877,6 +876,20 @@ where }; let rewritten = credential_span .in_scope(|| rewrite_http_header_block(&head, options.resolver)); + record_credential_preparation( + &credential_span, + "static", + "request_head", + match &rewritten { + Ok(result) if result.rewritten != head => "prepared", + Ok(_) => "not_needed", + Err(_) => "refused", + }, + ); + credential_span.record( + "otel.status_code", + if rewritten.is_ok() { "OK" } else { "ERROR" }, + ); drop(credential_span); let head = rewritten .map_err(|error| LiveWriteError::Failed(miette::Report::new(error)))? diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index d24a77ec34..6e5a440c8a 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -26,7 +26,7 @@ use openshell_ocsf::{ }; use tracing::{Instrument as _, warn}; -use crate::l7::correlation::{RequestCorrelation, RequestStage}; +use crate::l7::correlation::{RequestCorrelation, RequestStage, record_credential_preparation}; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; @@ -347,12 +347,31 @@ async fn inject_or_report( let result = match selected { Ok(credentials) => { inject_selected(req, ctx, credentials, request_id, correlation) - .instrument(span) + .instrument(span.clone()) .await } Err(error) => span.in_scope(|| Err(error)), }; + record_credential_preparation( + &span, + "token_grant", + "header", + if result.is_ok() { + "prepared" + } else { + "refused" + }, + ); + span.record( + "otel.status_code", + if result.is_ok() { "OK" } else { "ERROR" }, + ); if let Err(error) = &result { + span.record("openshell.credential.failure_code", error.failure().code()); + span.record( + "openshell.credential.failure_stage", + error.failure().stage().code(), + ); report_refused_grant(ctx, &method, &request_path, request_id, error, correlation); } if let Some((sender, context)) = ctx diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index be255a423b..f729225412 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -15,7 +15,7 @@ mod trace_tests; use openshell_sandbox_backend::test_support as rpc_test; use crate::identity::{BinaryIdentityCache, SuppliedIdentityError}; -use crate::l7::correlation::{HttpOutcome, RequestStage}; +use crate::l7::correlation::{HttpOutcome, RequestStage, record_credential_preparation}; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; use crate::policy_dns::{ @@ -5230,6 +5230,7 @@ fn rewrite_forward_request( Some(resolver) => secrets::rewrite_target_for_eval(path, resolver)?.resolved, None => path.to_string(), }; + let mut credential_prepared = upstream_path != path; let header_str = String::from_utf8_lossy(&raw[..header_end]); let lines = header_str.split("\r\n").collect::>(); @@ -5300,6 +5301,7 @@ fn rewrite_forward_request( Some(resolver) => rewrite_header_line_checked(line, resolver)?, None => line.to_string(), }; + credential_prepared |= rewritten_line != *line; output.extend_from_slice(rewritten_line.as_bytes()); output.extend_from_slice(b"\r\n"); @@ -5348,6 +5350,16 @@ fn rewrite_forward_request( return Err(secrets::UnresolvedPlaceholderError::unavailable("header")); } + record_credential_preparation( + &tracing::Span::current(), + "static", + "request_head", + if credential_prepared { + "prepared" + } else { + "not_needed" + }, + ); Ok(output) } @@ -6913,6 +6925,7 @@ async fn handle_forward_proxy_inner( }; if let Some(guard) = credential_generation { guard.ensure_current().inspect_err(|_| { + record_credential_preparation(&credential_stage, "static", "request_head", "stale"); egress::mark_error(&credential_stage); credential_stage.record( "openshell.request.failure_reason", @@ -6933,6 +6946,7 @@ async fn handle_forward_proxy_inner( ) }); if rewrite.is_err() { + record_credential_preparation(&credential_stage, "static", "request_head", "refused"); egress::mark_error(&credential_stage); credential_stage.record( "openshell.request.failure_reason", diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index 86aa709cec..ce5dfb90df 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -335,14 +335,17 @@ async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied ); let credential_failure = matches!(outcome, Outcome::UnresolvedCredential); - let secret_resolver = credential_failure.then(|| { - let (_, resolver) = SecretResolver::from_provider_env( - [("OTHER_KEY".into(), "inert-test-value".into())] - .into_iter() - .collect(), - ); - Arc::new(resolver.expect("test credential resolver")) - }); + let credential_prepared = matches!(outcome, Outcome::Allowed | Outcome::UpstreamClosed); + let secret_resolver = + (credential_failure || credential_prepared || matches!(outcome, Outcome::AuditDenied)) + .then(|| { + let (_, resolver) = SecretResolver::from_provider_env( + [("PRESENT".into(), "SENTINEL-CREDENTIAL".into())] + .into_iter() + .collect(), + ); + Arc::new(resolver.expect("test credential resolver")) + }); let mode = if matches!(outcome, Outcome::AuditDenied) { "audit" } else { @@ -399,6 +402,12 @@ network_policies: let target = format!("http://{ip}:{port}{path}?{query}"); let credential_header = if credential_failure { "Authorization: Bearer openshell:resolve:env:MISSING\r\n" + } else if matches!(outcome, Outcome::Allowed) { + "Authorization: Basic dXNlcjpvcGVuc2hlbGw6cmVzb2x2ZTplbnY6UFJFU0VOVA==\r\n" + } else if credential_prepared { + "Authorization: Bearer openshell:resolve:env:PRESENT\r\n" + } else if matches!(outcome, Outcome::AuditDenied) { + "Proxy-Authorization: Bearer openshell:resolve:env:PRESENT\r\nX-Removed: openshell:resolve:env:PRESENT\r\nConnection: x-removed\r\n" } else { "" }; @@ -458,6 +467,16 @@ network_policies: .starts_with(&format!("GET {path}?token=SENTINEL-QUERY HTTP/1.1\r\n")) ); assert!(!String::from_utf8_lossy(&bytes).contains("forged-receipt")); + assert_eq!( + String::from_utf8_lossy(&bytes).contains( + if matches!(outcome, Outcome::Allowed) { + "Basic dXNlcjpTRU5USU5FTC1DUkVERU5USUFM" + } else { + "Bearer SENTINEL-CREDENTIAL" + } + ), + credential_prepared, + ); if !matches!(outcome, Outcome::UpstreamClosed) { upstream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await.unwrap(); } @@ -672,12 +691,38 @@ network_policies: "credential_resolution_refused", ); } + if !policy_denied { + let preparation = stages + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .expect("selected resolver has a preparation stage"); + for (field, expected) in [ + ("openshell.credential.mechanism", "static"), + ("openshell.credential.placement", "request_head"), + ( + "openshell.credential.preparation", + if credential_failure { + "refused" + } else if credential_prepared { + "prepared" + } else { + "not_needed" + }, + ), + ] { + assert_eq!( + span_attribute(preparation, field).unwrap().as_str(), + expected + ); + } + } assert!( stages .iter() .all(|span| span.span_context.trace_id() == root.span_context.trace_id()) ); assert!(!format!("{spans:?}").contains("SENTINEL")); + assert!(!format!("{spans:?}").contains("dXNlcjpTRU5USU5FTC1DUkVERU5USUFM")); } capture.shutdown(); } diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index abe075e27b..ed73ef54f1 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -75,6 +75,8 @@ The CLI forwards `TRACEPARENT` and `TRACESTATE` from its environment on gateway The gateway also passes its trace context to compute drivers, so driver spans appear in the same trace as the gateway request that caused them. Built-in drivers export their spans through the gateway's process. Managed external drivers receive the trace endpoint, gateway name, and compute driver from the gateway. Drivers that you run yourself own their telemetry configuration. +Credential stages record `openshell.credential.mechanism`, `openshell.credential.placement` and `openshell.credential.preparation`, without credential values. Preparation can be `prepared`, `not_needed`, `refused`, `failed` or `stale`. `prepared` means resolution or signing succeeded, not that the upstream received the request. Token-grant failures also carry bounded failure codes and stages. + ## Next Steps - Configure [OTLP export](/how-it-works/gateways/configuration#otlp-export), including sampling and the `OTEL_*` variables the gateway honors.