diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index dd8af76169..e2e17299c3 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -94,3 +94,9 @@ filtering preserves raw HTTP bytes and borrows unchanged header blocks. It removes only protected receipt fields and their `Connection` or `Trailer` nominations, including folded continuations, without adding general response-header validation. + +Actual JSON-RPC/MCP body inspections have `openshell.mcp.inspect` child spans. +Each pass reports only its mode, selected revision, message shape and bounded +request/notification counts. Inspection acceptance does not imply policy +approval or remote tool execution. Method/tool names, arguments and payloads +are not added to these spans, and inspection adds no audit records. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index 1d4cfe36fa..867c0b9760 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -3,6 +3,8 @@ //! Request receipts and emission-time correlation without changing audit records. +pub mod jsonrpc; + use std::fmt; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc.rs b/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc.rs new file mode 100644 index 0000000000..49fe37c88f --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc.rs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded observations of actual JSON-RPC inspection passes, not execution. + +use super::{RequestTrace, current_request}; +use crate::l7::jsonrpc::{ + JsonRpcInspectionError, JsonRpcInspectionErrorKind, JsonRpcInspectionMode, + JsonRpcInspectionOptions, JsonRpcRequestInfo, +}; + +const MAX_RECORDED_CALLS: usize = 64; + +pub fn inspect( + options: JsonRpcInspectionOptions, + operation: impl FnOnce() -> JsonRpcRequestInfo, +) -> JsonRpcRequestInfo { + let span = current_request() + .filter(|scope| scope.inspected) + .and_then(|scope| { + scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map(RequestTrace::mcp_inspection_span) + }) + .unwrap_or_else(tracing::Span::none); + if span.is_disabled() { + return operation(); + } + + let info = span.in_scope(operation); + span.record( + "openshell.mcp.protocol", + match options.mode { + JsonRpcInspectionMode::Mcp => "mcp", + JsonRpcInspectionMode::JsonRpc => "jsonrpc", + }, + ); + if let Some(revision) = info.mcp_revision.or(options.mcp_revision) { + span.record("openshell.mcp.revision", revision.as_str()); + } + span.record( + "openshell.mcp.inspection.shape", + if info.receive_stream { + "receive_stream" + } else if info.is_batch { + "batch" + } else { + "single" + }, + ); + let (requests, notifications) = info.calls.iter().take(MAX_RECORDED_CALLS).fold( + (0_i64, 0_i64), + |(requests, notifications), call| { + if call.is_notification { + (requests, notifications + 1) + } else { + (requests + 1, notifications) + } + }, + ); + span.record("openshell.mcp.request_count", requests); + span.record("openshell.mcp.notification_count", notifications); + span.record("openshell.mcp.has_response", info.has_response); + span.record( + "openshell.mcp.counts_truncated", + info.calls.len() > MAX_RECORDED_CALLS, + ); + + let (outcome, failure) = match info.error.as_ref().map(JsonRpcInspectionError::kind) { + None => ("accepted", None), + // Transport selection may re-inspect under the selected profile. + Some(JsonRpcInspectionErrorKind::RevisionNotSelected) => ("revision_required", None), + Some(JsonRpcInspectionErrorKind::InvalidJson) => ("rejected", Some("invalid_json")), + Some(JsonRpcInspectionErrorKind::InvalidMessage) => ("rejected", Some("invalid_message")), + Some(JsonRpcInspectionErrorKind::McpProfileViolation) => { + ("rejected", Some("mcp_profile_violation")) + } + Some(JsonRpcInspectionErrorKind::McpLifecycleViolation) => { + ("rejected", Some("mcp_lifecycle_violation")) + } + }; + span.record("openshell.mcp.inspection.outcome", outcome); + if let Some(reason) = failure { + span.record("openshell.mcp.inspection.failure_kind", reason); + span.record("openshell.request.failure_reason", reason); + span.record("otel.status_code", "ERROR"); + } else { + span.record("otel.status_code", "OK"); + } + info +} + +pub fn receive_stream(options: JsonRpcInspectionOptions) -> JsonRpcRequestInfo { + inspect(options, JsonRpcRequestInfo::receive_stream) +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc/tests.rs new file mode 100644 index 0000000000..2084d53518 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation/jsonrpc/tests.rs @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, scope_connection}; +use crate::l7::jsonrpc::parse_jsonrpc_body_with_options; +use crate::l7::relay::L7EvalContext; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use opentelemetry_sdk::trace::Sampler; +use tracing::instrument::WithSubscriber as _; + +#[tokio::test] +async fn inspection_metadata_is_bounded_and_does_not_expose_parser_input() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let options = JsonRpcInspectionMode::JsonRpc.into(); + let mut messages = (0..70) + .map(|id| { + let mut message = serde_json::json!({ + "jsonrpc": "2.0", "method": "DO-NOT-RECORD-METHOD", + "params": { "secret": "DO-NOT-RECORD-PAYLOAD" } + }); + if id % 2 == 0 { + message["id"] = id.into(); + } + message + }) + .collect::>(); + messages.push(serde_json::json!({"jsonrpc":"2.0", "id":123, "result":"DO-NOT-RECORD-RESULT"})); + let body = serde_json::to_vec(&messages).unwrap(); + scope_connection(async { + let correlation = RequestCorrelation::new_http( + "POST", + b"POST /rpc HTTP/1.1\r\nHost: example.test\r\n\r\n", + &L7EvalContext::default(), + ); + let info = parse_jsonrpc_body_with_options(&body, options); + assert_eq!(info.calls.len(), 70); + assert_eq!(info.calls[0].method, "DO-NOT-RECORD-METHOD"); + assert!(info.has_response); + let malformed = parse_jsonrpc_body_with_options( + br#"{"DO-NOT-RECORD-KEY":1,"DO-NOT-RECORD-KEY":2}"#, + options, + ); + assert_eq!( + malformed.error.as_ref().unwrap().kind(), + JsonRpcInspectionErrorKind::InvalidJson + ); + correlation.record_outcome(HttpOutcome::Allowed); + }) + .with_subscriber(capture.subscriber()) + .await; + + // The public parser is also used without an inspected request. + tracing::subscriber::with_default(capture.subscriber(), || { + assert_eq!( + parse_jsonrpc_body_with_options(&body, options).calls.len(), + 70 + ); + }); + let spans = capture.spans(); + let inspections = spans + .iter() + .filter(|span| span.name == "openshell.mcp.inspect") + .collect::>(); + assert_eq!(inspections.len(), 2); + let batch = inspections + .iter() + .find(|span| { + span_attribute(span, "openshell.mcp.inspection.shape") + .is_some_and(|value| value.as_str() == "batch") + }) + .unwrap(); + for (key, value) in [ + ("openshell.mcp.request_count", 32_i64), + ("openshell.mcp.notification_count", 32_i64), + ] { + assert_eq!( + span_attribute(batch, key), + Some(&opentelemetry::Value::I64(value)) + ); + } + assert_eq!( + span_attribute(batch, "openshell.mcp.has_response"), + Some(&opentelemetry::Value::Bool(true)) + ); + assert_eq!( + span_attribute(batch, "openshell.mcp.counts_truncated"), + Some(&opentelemetry::Value::Bool(true)) + ); + let rejected = inspections + .iter() + .find(|span| { + span_attribute(span, "openshell.mcp.inspection.outcome") + .is_some_and(|value| value.as_str() == "rejected") + }) + .unwrap(); + assert_eq!( + span_attribute(rejected, "openshell.mcp.inspection.failure_kind") + .unwrap() + .as_str(), + "invalid_json" + ); + assert!(matches!( + rejected.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD")); + assert!(capture.events.json_events().is_empty()); + capture.shutdown(); +} diff --git a/crates/openshell-supervisor-network/src/l7/jsonrpc.rs b/crates/openshell-supervisor-network/src/l7/jsonrpc.rs index 6d767191fd..dbd1e6dc81 100644 --- a/crates/openshell-supervisor-network/src/l7/jsonrpc.rs +++ b/crates/openshell-supervisor-network/src/l7/jsonrpc.rs @@ -18,7 +18,7 @@ use tower_mcp_types::{ protocol::{InitializeParams, JSONRPC_VERSION, McpRequest, RequestMeta, validate_meta_object}, }; -use crate::l7::provider::{L7Provider, L7Request}; +use crate::l7::provider::L7Request; pub const DEFAULT_MAX_BODY_BYTES: usize = 64 * 1024; @@ -114,26 +114,19 @@ pub struct JsonRpcHttpRequest { pub info: JsonRpcRequestInfo, } -pub(crate) async fn parse_jsonrpc_http_request( +pub(crate) async fn inspect_jsonrpc_http_request_body( client: &mut C, + request: &mut L7Request, max_body_bytes: usize, - canonicalize_options: crate::l7::path::CanonicalizeOptions, inspection_options: JsonRpcInspectionOptions, -) -> Result> { - let provider = crate::l7::rest::RestProvider::with_options(canonicalize_options); - let Some(mut request) = provider.parse_request(client).await? else { - return Ok(None); - }; - if jsonrpc_receive_stream_request(&request) { - return Ok(Some(JsonRpcHttpRequest { - request, - info: JsonRpcRequestInfo::receive_stream(), - })); +) -> Result { + if jsonrpc_receive_stream_request(request) { + return Ok(crate::l7::correlation::jsonrpc::receive_stream( + inspection_options, + )); } - let body = - crate::l7::http::read_body_for_inspection(client, &mut request, max_body_bytes).await?; - let info = parse_jsonrpc_body_with_options(&body, inspection_options); - Ok(Some(JsonRpcHttpRequest { request, info })) + let body = crate::l7::http::read_body_for_inspection(client, request, max_body_bytes).await?; + Ok(parse_jsonrpc_body_with_options(&body, inspection_options)) } /// Reinspect a fully buffered JSON-RPC-family HTTP request after middleware. @@ -148,7 +141,9 @@ pub(crate) fn inspect_buffered_jsonrpc_http_request( inspection_options: JsonRpcInspectionOptions, ) -> Result { if jsonrpc_receive_stream_request(request) { - return Ok(JsonRpcRequestInfo::receive_stream()); + return Ok(crate::l7::correlation::jsonrpc::receive_stream( + inspection_options, + )); } let header_end = request @@ -416,6 +411,15 @@ pub fn parse_jsonrpc_body( pub fn parse_jsonrpc_body_with_options( body: &[u8], inspection_options: JsonRpcInspectionOptions, +) -> JsonRpcRequestInfo { + crate::l7::correlation::jsonrpc::inspect(inspection_options, || { + parse_jsonrpc_body_inner(body, inspection_options) + }) +} + +fn parse_jsonrpc_body_inner( + body: &[u8], + inspection_options: JsonRpcInspectionOptions, ) -> JsonRpcRequestInfo { let value = match parse_unique_json_value(body) { Ok(value) => value, diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 9725ef0ea5..5ffea55264 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1684,7 +1684,9 @@ where }; let mut jsonrpc_info = if config.protocol.is_jsonrpc_family() { if crate::l7::jsonrpc::jsonrpc_receive_stream_request(&req) { - Some(crate::l7::jsonrpc::JsonRpcRequestInfo::receive_stream()) + Some(crate::l7::correlation::jsonrpc::receive_stream( + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + )) } else { match crate::l7::http::read_body_for_inspection( client, @@ -3109,17 +3111,33 @@ where .endpoint_observation_tx .as_ref() .and_then(EndpointObservationSender::capture); - let parsed = match crate::l7::jsonrpc::parse_jsonrpc_http_request( - client, - config.json_rpc_max_body_bytes, - crate::l7::path::CanonicalizeOptions { + let provider = + crate::l7::rest::RestProvider::with_options(crate::l7::path::CanonicalizeOptions { allow_encoded_slash: config.allow_encoded_slash, ..Default::default() - }, - crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), - ) - .await - { + }); + // Retain the scope while body-parse failures are audited. + let mut correlation = None; + let parsed: Result<_> = async { + let Some(mut request) = provider.parse_request(client).await? else { + return Ok(None); + }; + let request_correlation = RequestCorrelation::new(&request, ctx); + correlation = Some(request_correlation); + let info = crate::l7::jsonrpc::inspect_jsonrpc_http_request_body( + client, + &mut request, + config.json_rpc_max_body_bytes, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + ) + .await?; + Ok(Some(crate::l7::jsonrpc::JsonRpcHttpRequest { + request, + info, + })) + } + .await; + let parsed = match parsed { Ok(Some(parsed)) => parsed, Ok(None) => return Ok(()), Err(e) => { @@ -3141,7 +3159,7 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; - let correlation = RequestCorrelation::new(&req, ctx); + let correlation = correlation.expect("a parsed request has a correlation scope"); let capture_error = |error| correlation.capture_error(error); let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index 4128083134..51009b31a7 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -1237,6 +1237,92 @@ async fn production_single_mcp_policy_deny_correlates_record_and_receipt() { ); } +#[tokio::test] +async fn mcp_inspection_passes_join_real_policy_outcomes_without_recording_payloads() { + let initialize = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"DO-NOT-RECORD-CLIENT","version":"1"}}}"#; + let denied = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; + for (sampled, body, allowed, status) in [ + (true, initialize, true, 200), + (true, denied, false, 403), + (true, "{DO-NOT-RECORD-BODY", false, 400), + (false, denied, false, 403), + ] { + let evidence = capture_request_spans( + sampled, + Box::pin(run_exchange( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 2025-11-25\r\n", + body, + ), + allowed.then_some( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}" + .as_slice(), + ), + )), + ) + .await; + evidence.assert_no_deny_span(sampled); + assert_eq!(!evidence.output.forwarded.is_empty(), allowed); + assert!( + evidence + .output + .response + .starts_with(format!("HTTP/1.1 {status} ").as_bytes()), + "{body}: {}", + String::from_utf8_lossy(&evidence.output.response), + ); + let receipt = request_id(&evidence.output.response); + assert!(!evidence.events.is_empty()); + for event in &evidence.events { + assert_eq!(event["metadata"]["correlation_uid"], receipt); + } + if sampled { + let root = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + if allowed { "allowed" } else { "denied" } + ); + let inspections = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.mcp.inspect") + .collect::>(); + assert!(!inspections.is_empty()); + for inspection in &inspections { + assert_eq!(inspection.parent_span_id, root.span_context.span_id()); + assert!(inspection.start_time >= root.start_time); + assert!(inspection.end_time <= root.end_time); + assert_eq!( + span_attribute(inspection, "openshell.mcp.protocol") + .unwrap() + .as_str(), + "mcp" + ); + } + // A valid request can be rejected by policy after inspection accepted it. + let expected = if body.starts_with("{DO-NOT") { + "rejected" + } else { + "accepted" + }; + assert!(inspections.iter().any(|span| { + span_attribute(span, "openshell.mcp.inspection.outcome") + .is_some_and(|value| value.as_str() == expected) + })); + assert!(!format!("{:?}", evidence.spans).contains("DO-NOT-RECORD")); + } + } +} + #[tokio::test] async fn production_single_graphql_allow_correlates_record_and_response_receipt() { let graphql = fixture( diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs index 1d9c36a509..53a737529d 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -104,6 +104,27 @@ impl RequestTrace { ) } + pub(super) fn mcp_inspection_span(&self) -> Span { + if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() { + return Span::none(); + } + tracing::info_span!( + parent: &self.span, + "openshell.mcp.inspect", + otel.status_code = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + openshell.mcp.protocol = tracing::field::Empty, + openshell.mcp.revision = tracing::field::Empty, + openshell.mcp.inspection.shape = tracing::field::Empty, + openshell.mcp.inspection.outcome = tracing::field::Empty, + openshell.mcp.inspection.failure_kind = tracing::field::Empty, + openshell.mcp.request_count = tracing::field::Empty, + openshell.mcp.notification_count = tracing::field::Empty, + openshell.mcp.has_response = tracing::field::Empty, + openshell.mcp.counts_truncated = tracing::field::Empty, + ) + } + pub(super) fn record_status(&self, status: u16) { self.span .record("http.response.status_code", i64::from(status)); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index be255a423b..d266f5fa9e 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -6279,7 +6279,9 @@ async fn handle_forward_proxy_inner( body_length, }; let info = if crate::l7::jsonrpc::jsonrpc_receive_stream_request(&jsonrpc_request) { - crate::l7::jsonrpc::JsonRpcRequestInfo::receive_stream() + crate::l7::correlation::jsonrpc::receive_stream( + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(&l7_config.config), + ) } else { let body = match crate::l7::http::read_body_for_inspection( client, diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index abe075e27b..b3fdb7332c 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -47,6 +47,14 @@ Each component names its spans after the operation it performs: The gateway's client span and the driver's server span for a compute-driver call share the same name, which also appears in `rpc.method`. Their service names and span kinds tell them apart. A driver watch stream records a terminal status when it observes one. If the gateway stops consuming the stream first, the span status stays unset. +For inspected JSON-RPC and MCP requests, `openshell.mcp.inspect` measures each +actual body-inspection pass. It records the protocol, selected revision, message +shape, inspection outcome and counts from at most 64 calls or notifications. +`counts_truncated` indicates a larger batch. Names, arguments and payloads are +not added to these spans. An accepted inspection can still be denied by policy; +`revision_required` means the transport must select a revision before inspection +can finish. These spans do not establish that an upstream tool executed. + ## Resource Attributes Resource attributes identify which process produced a span: