diff --git a/Cargo.lock b/Cargo.lock index 458b6000aa..e9de3401a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5118,6 +5118,7 @@ dependencies = [ "libc", "metrics", "metrics-exporter-prometheus", + "metrics-util", "miette", "nix 0.29.0", "notify", diff --git a/Cargo.toml b/Cargo.toml index 5254c11284..55521c155c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ tracing-opentelemetry = { version = "0.34", default-features = false, features = # Metrics metrics = "0.24" +metrics-util = { version = "0.20", default-features = false } metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] } # Unix/Process diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index d1330a299e..1f64a329a2 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -942,6 +942,7 @@ mod tests { .unwrap(); assert!(collector.tls().client_identity().is_some()); let otlp = OtlpConfig { + metrics: None, service_name: None, traces: Some(OtlpTracesConfig { collector }), ocsf: None, @@ -974,6 +975,7 @@ mod tests { append_otlp_args( &mut command, Some(&OtlpConfig { + metrics: None, service_name: Some("custom-gateway".to_string()), traces: Some(OtlpTracesConfig { collector: OtlpCollectorConfig::new( @@ -1006,6 +1008,7 @@ mod tests { append_otlp_args( &mut command, Some(&OtlpConfig { + metrics: None, service_name: None, traces: Some(OtlpTracesConfig { collector: OtlpCollectorConfig::new( @@ -1040,6 +1043,7 @@ mod tests { append_otlp_args( &mut command, Some(&OtlpConfig { + metrics: None, service_name: None, traces: None, ocsf: Some(OcsfOtlpConfig { @@ -1078,6 +1082,7 @@ mod tests { append_otlp_args( &mut command, Some(&OtlpConfig { + metrics: None, service_name: None, traces: None, ocsf: None, diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml index d14fe33db8..b562a8b7ff 100644 --- a/crates/openshell-otel-test-support/Cargo.toml +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -18,7 +18,7 @@ hyper = { workspace = true } hyper-util = { workspace = true, features = ["tokio"] } openshell-crypto = { workspace = true } openshell-otel = { path = "../openshell-otel" } -opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace"] } +opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace", "metrics"] } prost = { workspace = true } rcgen = { workspace = true } rustls = { workspace = true } diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index 13f4e49928..a9f3ad3e6b 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -74,6 +74,10 @@ use opentelemetry_proto::tonic::collector::logs::v1::{ ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::{LogsService, LogsServiceServer}, }; +use opentelemetry_proto::tonic::collector::metrics::v1::{ + ExportMetricsServiceRequest, ExportMetricsServiceResponse, + metrics_service_server::{MetricsService, MetricsServiceServer}, +}; use opentelemetry_proto::tonic::collector::trace::v1::{ ExportTraceServiceRequest, ExportTraceServiceResponse, trace_service_server::{TraceService, TraceServiceServer}, @@ -141,6 +145,10 @@ pub async fn assert_compute_driver_tracing( #[derive(Clone, Debug, Default)] pub struct ReceivedTraces { + /// Raw metric export requests, including resources and aggregation data. + pub metric_exports: Vec, + /// `authorization` metadata of each metric export request. + pub metric_authorizations: Vec, pub spans: Vec, /// String bodies of exported log records. pub log_bodies: Vec, @@ -281,6 +289,38 @@ impl LogsService for Collector { } } +#[tonic::async_trait] +impl MetricsService for Collector { + async fn export( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + { + let mut received = self + .received + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(authorization) = request + .metadata() + .get("authorization") + .and_then(|value| value.to_str().ok()) + { + received + .metric_authorizations + .push(authorization.to_string()); + } + let request = request.into_inner(); + for metrics in &request.resource_metrics { + let attributes = received.record_resource(metrics.resource.clone()); + received.resource_attributes.push(attributes); + } + received.metric_exports.push(request); + } + self.exported.notify_one(); + Ok(tonic::Response::new(ExportMetricsServiceResponse::default())) + } +} + /// Loopback OTLP/gRPC server that captures exported spans, logs, and resources. pub struct OtlpTestServer { endpoint: String, @@ -353,13 +393,16 @@ impl OtlpTestServer { } let trace_service = TraceServiceServer::new(collector.clone()) .accept_compressed(tonic::codec::CompressionEncoding::Gzip); - let logs_service = LogsServiceServer::new(collector) + let logs_service = LogsServiceServer::new(collector.clone()) + .accept_compressed(tonic::codec::CompressionEncoding::Gzip); + let metrics_service = MetricsServiceServer::new(collector) .accept_compressed(tonic::codec::CompressionEncoding::Gzip); let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel(); let task = tokio::spawn(async move { server .add_service(trace_service) .add_service(logs_service) + .add_service(metrics_service) .serve_with_incoming_shutdown( tokio_stream::wrappers::TcpListenerStream::new(listener), async { diff --git a/crates/openshell-otel/Cargo.toml b/crates/openshell-otel/Cargo.toml index a857018b91..95d34ed5b1 100644 --- a/crates/openshell-otel/Cargo.toml +++ b/crates/openshell-otel/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openshell-otel" -description = "Shared OpenTelemetry trace and log export support for OpenShell services" +description = "Shared OpenTelemetry export support for OpenShell services" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -24,7 +24,7 @@ openshell-ocsf = { path = "../openshell-ocsf", optional = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } opentelemetry-http = { workspace = true, optional = true } -opentelemetry-otlp = { workspace = true, optional = true, features = ["gzip-http", "http-proto", "logs"] } +opentelemetry-otlp = { workspace = true, optional = true, features = ["gzip-http", "http-proto", "logs", "metrics"] } rustls = { workspace = true, optional = true } rustls-native-certs = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } @@ -39,7 +39,7 @@ zeroize = { workspace = true, optional = true } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } -opentelemetry-proto = { workspace = true, features = ["logs"] } +opentelemetry-proto = { workspace = true, features = ["logs", "metrics"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost = { workspace = true } tempfile = "3" diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 32077a7f93..b8c2f4d79b 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -1,14 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Shared OpenTelemetry trace and log export support for `OpenShell` services. +//! Shared OpenTelemetry export support for `OpenShell` services. //! //! The crate has two tiers: //! //! - Trace-context propagation, always compiled in: W3C `traceparent` / //! `tracestate` extraction and injection for HTTP headers, tonic metadata, //! and process environments. -//! - The `export` feature (on by default): OTLP tracer providers, tracing +//! - The `export` feature (on by default): OTLP tracer and meter providers, tracing //! layers, compute-driver and gRPC tracing machinery, and the OCSF log //! bridge. Services that report to a collector keep the default; clients //! that only forward trace context (e.g. the CLI) opt out with @@ -35,6 +35,8 @@ mod headers; #[cfg(feature = "export")] mod logs; #[cfg(feature = "export")] +mod metrics; +#[cfg(feature = "export")] mod ocsf; #[cfg(feature = "export")] mod setup; diff --git a/crates/openshell-otel/src/metrics.rs b/crates/openshell-otel/src/metrics.rs new file mode 100644 index 0000000000..aa59f50e23 --- /dev/null +++ b/crates/openshell-otel/src/metrics.rs @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OTLP metric export support. + +use opentelemetry_otlp::MetricExporter; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; + +use crate::{OtlpTarget, Protocol, SetupError}; + +impl OtlpTarget { + /// Build a meter provider that periodically exports to this collector. + pub fn meter_provider(&self) -> Result { + self.check_tls()?; + let exporter = match self.protocol { + Protocol::Grpc => self + .configure(MetricExporter::builder().with_tonic()) + .build()?, + Protocol::HttpProtobuf => self + .configure_http( + MetricExporter::builder().with_http(), + "v1/metrics", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + )? + .build()?, + }; + Ok(SdkMeterProvider::builder() + .with_reader(PeriodicReader::builder(exporter).build()) + .with_resource(self.resource().clone()) + .build()) + } +} + +#[cfg(test)] +mod tests { + use openshell_otel_test_support::{OtlpHttpTestServer, OtlpTestServer, ServerTls, TestPki}; + use opentelemetry::{KeyValue, metrics::MeterProvider as _}; + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::{metric, number_data_point}; + use prost::Message as _; + + use super::*; + use crate::{OtlpExportConfig, OtlpTls, ServiceName}; + + fn target(endpoint: &str, protocol: Protocol) -> OtlpTarget { + OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("metrics-test"), + service_version: Some("1.2.3"), + resource_attributes: vec![KeyValue::new("openshell.gateway.name", "test-gateway")], + }) + .unwrap() + .with_protocol(protocol) + } + + fn assert_counter(request: &ExportMetricsServiceRequest) { + let resource = &request.resource_metrics[0]; + assert!( + resource + .resource + .as_ref() + .unwrap() + .attributes + .iter() + .any(|attribute| attribute.key == "service.name" + && format!("{:?}", attribute.value).contains("metrics-test")) + ); + let metric = &resource.scope_metrics[0].metrics[0]; + assert_eq!(metric.name, "requests_total"); + let Some(metric::Data::Sum(sum)) = &metric.data else { + panic!("expected counter sum") + }; + assert!(sum.is_monotonic); + assert_eq!( + sum.data_points[0].value, + Some(number_data_point::Value::AsInt(7)) + ); + assert_eq!(sum.data_points[0].attributes[0].key, "method"); + } + + #[tokio::test] + async fn metrics_reuse_headers_and_mutual_tls_over_both_protocols() { + crate::test_environment::Environment::new() + .remove("OTEL_EXPORTER_OTLP_HEADERS") + .remove("OTEL_EXPORTER_OTLP_METRICS_HEADERS") + .run_async(async { + let pki = TestPki::generate(); + let server_tls = ServerTls::mutual(&pki); + let tls = OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + for protocol in [Protocol::Grpc, Protocol::HttpProtobuf] { + let grpc = OtlpTestServer::start_with_tls(&server_tls).await; + let mut http = OtlpHttpTestServer::start_with_tls(&server_tls).await; + let endpoint = if protocol == Protocol::Grpc { + grpc.endpoint() + } else { + http.endpoint() + }; + let provider = target(endpoint, protocol) + .with_tls(tls.clone()) + .with_headers(crate::parse_otlp_headers("authorization=Bearer%20metrics")) + .meter_provider() + .unwrap(); + provider + .meter("test") + .u64_counter("requests_total") + .build() + .add(7, &[KeyValue::new("method", "GET")]); + let flush = { + let provider = provider.clone(); + tokio::task::spawn_blocking(move || provider.force_flush()) + }; + flush.await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || provider.shutdown()) + .await + .unwrap() + .unwrap(); + if protocol == Protocol::Grpc { + let received = grpc.shutdown().await; + assert_eq!(received.metric_authorizations[0], "Bearer metrics"); + assert_counter(&received.metric_exports[0]); + } else { + let export = http.next_export().await; + assert_eq!(export.path, "/v1/metrics"); + assert_eq!(export.headers["authorization"], "Bearer metrics"); + assert_counter(&ExportMetricsServiceRequest::decode(export.body).unwrap()); + grpc.shutdown().await; + } + } + }) + .await; + } + + #[tokio::test] + async fn metrics_export_periodically_without_scraping_and_flush_on_shutdown() { + crate::test_environment::Environment::new() + .set("OTEL_METRIC_EXPORT_INTERVAL", "25") + .set("OTEL_EXPORTER_OTLP_METRICS_HEADERS", "x-metrics=from-env") + .set( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "http://ignored.invalid:4318", + ) + .run_async(async { + let mut collector = OtlpHttpTestServer::start().await; + let provider = target( + &format!("{}/prefix", collector.endpoint()), + Protocol::HttpProtobuf, + ) + .meter_provider() + .unwrap(); + let counter = provider.meter("test").u64_counter("requests_total").build(); + counter.add(7, &[KeyValue::new("method", "GET")]); + for _ in 0..2 { + let export = collector.next_export().await; + assert_eq!(export.path, "/prefix/v1/metrics"); + assert_eq!(export.headers["x-metrics"], "from-env"); + assert_counter(&ExportMetricsServiceRequest::decode(export.body).unwrap()); + } + tokio::task::spawn_blocking(move || provider.shutdown()) + .await + .unwrap() + .unwrap(); + collector.next_export().await; + }) + .await; + } +} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 9d7f9299db..703aa954bf 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -72,7 +72,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } -# OpenTelemetry (OTLP trace and OCSF log export, opt-in via [openshell.gateway.otlp]) +# OpenTelemetry (OTLP export, opt-in via [openshell.gateway.otlp]) opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } tracing-opentelemetry = { workspace = true } @@ -80,6 +80,7 @@ tracing-opentelemetry = { workspace = true } # Metrics metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +metrics-util = { workspace = true, features = ["registry"] } # Utilities base64 = { workspace = true } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index e3c8b27186..7a366b8660 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -922,8 +922,15 @@ async fn run_from_args( &prepared.compute_driver, &prepared.config.compute_driver_endpoints, ); - let (otlp_export, otlp_error) = + let (mut otlp_export, otlp_error) = crate::otel_tracing::OtlpExport::start(otlp_config, gateway_resource); + let (metrics_recorder, metrics_error) = match (otlp_export.as_mut(), otlp_config) { + (Some(export), Some(config)) => match export.start_metrics(config, gateway_resource) { + Ok(recorder) => (recorder, None), + Err(error) => (None, Some(error)), + }, + _ => (None, None), + }; let (tracing_handle, driver_otlp_error) = crate::tracing_setup::install( &EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)) @@ -990,6 +997,9 @@ async fn run_from_args( if has_oidc { info!("OIDC authentication enabled"); } + if let Some(error) = metrics_error { + error!(%error, "OTLP metrics could not be started; continuing without metric export"); + } if let Some(err) = otlp_error.as_ref().or(driver_otlp_error.as_ref()) { error!( error = %err, @@ -1005,6 +1015,7 @@ async fn run_from_args( ocsf_protocol = otlp.ocsf_protocol().map(tracing::field::display), agent_endpoint = otlp.agent_endpoint().map(tracing::field::display), agent_protocol = otlp.agent_protocol().map(tracing::field::display), + metrics_endpoint = otlp.metrics_endpoint().map(tracing::field::display), "OTLP exporting enabled" ); } @@ -1014,7 +1025,13 @@ async fn run_from_args( info!(bind = %prepared.config.bind_address, "Starting OpenShell server"); - let result = Box::pin(run_server(prepared, tracing_log_bus, compute_drivers)).await; + let result = Box::pin(run_server( + prepared, + tracing_log_bus, + compute_drivers, + metrics_recorder, + )) + .await; tracing_handle.shutdown(); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index bd04a38eef..b0f1972914 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -362,11 +362,12 @@ pub struct MetricsTlsFileConfig { /// `[openshell.gateway.otlp]` section and its per-signal tables. /// /// `endpoint` is the collector for every enabled signal, and on its own -/// enables trace export. `[otlp.traces]` overrides the trace endpoint, -/// `[otlp.ocsf]` with `enabled = true` exports OCSF events as log records, and -/// `[otlp.agent]` with `enabled = true` relays workload traces, each -/// optionally to a different endpoint. SDK tuning knobs are deliberately absent — see -/// [`crate::otel_tracing`] for what this table owns and what the `OTEL_*` +/// enables trace export. `[otlp.traces]` overrides the trace endpoint; +/// `[otlp.metrics]` with `enabled = true` exports gateway metrics; +/// `[otlp.ocsf]` with `enabled = true` exports OCSF events as log records; and +/// `[otlp.agent]` with `enabled = true` relays workload traces. Each signal +/// can use a different endpoint. SDK tuning knobs are deliberately absent — +/// see [`crate::otel_tracing`] for what this table owns and what the `OTEL_*` /// environment variables own. /// /// Endpoints are resolved when parsing, so each enabled signal carries its own. @@ -381,6 +382,14 @@ pub struct OtlpConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub traces: Option, + /// Gateway metric export, when enabled. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_enabled" + )] + pub metrics: Option, + /// OCSF event export as OTLP log records, when enabled. #[serde( default, @@ -865,6 +874,8 @@ pub struct AgentOtlpConfig { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawOtlpConfig { + #[serde(default)] + metrics: Option, #[serde(default)] endpoint: Option, #[serde(default)] @@ -910,6 +921,27 @@ struct RawAgentOtlp { key_file: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawOtlpMetrics { + #[serde(default)] + enabled: bool, + #[serde(default)] + endpoint: Option, + #[serde(default)] + protocol: Option, + #[serde(default)] + headers: Option>, + #[serde(default)] + headers_file: Option, + #[serde(default)] + ca_file: Option, + #[serde(default)] + cert_file: Option, + #[serde(default)] + key_file: Option, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawOtlpTraces { @@ -1045,6 +1077,11 @@ impl TryFrom for OtlpConfig { } (None, None) => None, }; + let metrics = raw + .metrics + .filter(|metrics| metrics.enabled) + .map(|metrics| raw_collector!(metrics).resolve("otlp.metrics", shared.as_ref())) + .transpose()?; let traces = match raw.traces { Some(traces) => Some(raw_collector!(traces).resolve("otlp.traces", shared.as_ref())?), None => shared.clone(), @@ -1069,14 +1106,12 @@ impl TryFrom for OtlpConfig { }) }) .transpose()?; - if traces.is_none() && ocsf.is_none() && agent.is_none() { - return Err( - "otlp needs an endpoint, or otlp.ocsf or otlp.agent with enabled = true" - .to_string(), - ); + if traces.is_none() && metrics.is_none() && ocsf.is_none() && agent.is_none() { + return Err("otlp needs an endpoint, or an enabled signal table".to_string()); } let config = Self { service_name: raw.service_name, + metrics, traces, ocsf, agent, @@ -2176,6 +2211,57 @@ service_name = "openshell-gateway-dev" assert_eq!(otlp.traces_endpoint(), Some("http://127.0.0.1:4317")); assert!(otlp.service_name.is_none()); assert_eq!(otlp.ocsf, None); + assert!(otlp.metrics.is_none()); + } + + #[test] + fn otlp_metrics_use_the_shared_collector_and_roundtrip() { + let config = otlp( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4318'\n\ + protocol = 'http/protobuf'\n\ + headers = { authorization = 'Bearer shared' }\n\n\ + [openshell.gateway.otlp.metrics]\nenabled = true\n\n\ + [openshell.gateway.otlp.traces]\nendpoint = 'http://traces:4317'\n", + ); + let metrics = config.metrics.as_ref().unwrap(); + assert_eq!(metrics.endpoint, "http://shared:4318"); + assert_eq!(metrics.protocol, OtlpProtocol::HttpProtobuf); + assert_eq!( + metrics.headers.header_map()["authorization"], + "Bearer shared" + ); + assert_eq!(config.traces_endpoint(), Some("http://traces:4317")); + let encoded = toml::to_string(&config).unwrap(); + assert!(encoded.contains("[metrics]\nenabled = true")); + assert_eq!(toml::from_str::(&encoded).unwrap(), config); + assert_eq!( + config.supervisor_otlp().traces.unwrap().endpoint, + "http://traces:4317" + ); + } + + #[test] + fn otlp_metrics_are_opt_in_and_can_use_their_own_endpoint() { + let disabled = otlp( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\n\ + [openshell.gateway.otlp.metrics]\n", + ); + assert!(disabled.metrics.is_none()); + + let own = otlp( + "[openshell.gateway.otlp.metrics]\nenabled = true\n\ + endpoint = 'http://metrics:4318'\nprotocol = 'http/protobuf'\n", + ); + let metrics = own.metrics.as_ref().unwrap(); + assert_eq!(metrics.endpoint, "http://metrics:4318"); + assert_eq!(metrics.protocol, OtlpProtocol::HttpProtobuf); + assert!(own.traces.is_none()); + + let error = toml::from_str::( + "[metrics]\nenabled = true\n[traces]\nendpoint = 'http://traces:4317'\n", + ) + .unwrap_err(); + assert!(error.to_string().contains("otlp.metrics needs an endpoint")); } #[test] diff --git a/crates/openshell-server/src/gateway_metrics.rs b/crates/openshell-server/src/gateway_metrics.rs index 6e907696c7..c8005a0392 100644 --- a/crates/openshell-server/src/gateway_metrics.rs +++ b/crates/openshell-server/src/gateway_metrics.rs @@ -8,7 +8,7 @@ //! channel, endpoint, token, or replica id. The scrape target already identifies the replica. //! //! Metric handles bind to whichever recorder is current when a macro runs. `run_server` -//! installs the Prometheus recorder after it builds `ServerState`, so never cache a handle in a +//! installs the recorder after it builds `ServerState`, so never cache a handle in a //! static or in state built before [`install_global_recorder`]. [`GaugeSlot`] acquires its //! handle when the tracked object is created and releases it through the same handle, so a //! slot can never drive a series negative. @@ -19,8 +19,12 @@ use metrics::{ Gauge, Unit, counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, }; use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder, PrometheusHandle}; +use metrics_util::layers::FanoutBuilder; use tonic::{Code, Status}; +mod otlp_recorder; +pub use otlp_recorder::OtlpRecorder; + // Gauges pub const SUPERVISOR_SESSIONS: &str = "openshell_server_supervisor_sessions"; pub const SUPERVISOR_HOSTS: &str = "openshell_server_supervisor_hosts"; @@ -247,10 +251,38 @@ pub fn configure_exporter(builder: PrometheusBuilder) -> Result, relay: RelayCapacity, mutation_lock_connections: Option, -) -> Result { - let handle = configure_exporter(PrometheusBuilder::new())?.install_recorder()?; +) -> anyhow::Result> { + if !prometheus && otlp.is_none() { + return Ok(None); + } + let recorder = prometheus + .then(|| { + configure_exporter(PrometheusBuilder::new()).map(PrometheusBuilder::build_recorder) + }) + .transpose()?; + let handle = recorder + .as_ref() + .map(metrics_exporter_prometheus::PrometheusRecorder::handle); + match (recorder, otlp) { + (Some(prometheus), Some(otlp)) => metrics::set_global_recorder( + FanoutBuilder::default() + .add_recorder(prometheus) + .add_recorder(otlp) + .build(), + ) + .map_err(|error| anyhow::anyhow!("{error}"))?, + (Some(prometheus), None) => { + metrics::set_global_recorder(prometheus).map_err(|error| anyhow::anyhow!("{error}"))?; + } + (None, Some(otlp)) => { + metrics::set_global_recorder(otlp).map_err(|error| anyhow::anyhow!("{error}"))?; + } + (None, None) => unreachable!(), + } describe_and_initialize(relay, mutation_lock_connections); Ok(handle) } @@ -856,6 +888,85 @@ mod tests { ); } + #[test] + fn recorder_modes_preserve_gateway_metric_values() { + use opentelemetry::metrics::MeterProvider as _; + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ScopeMetrics}; + use opentelemetry_sdk::metrics::{ + InMemoryMetricExporter, PeriodicReader, SdkMeterProvider, + }; + + for (prometheus, otlp) in [(false, false), (true, false), (false, true), (true, true)] { + crate::test_environment::Environment::new().run(|| { + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_reader(PeriodicReader::builder(exporter.clone()).build()) + .build(); + let recorder = otlp.then(|| OtlpRecorder::new(provider.meter("test"))); + let handle = install_global_recorder(prometheus, recorder, RelayCapacity { per_replica: 256 }, None).unwrap(); + let requests = counter!("openshell_server_grpc_requests_total", "method" => "ListSandboxes", "code" => "0"); + requests.increment(2); + requests.absolute(3); + requests.absolute(2); + let pending = gauge!(RELAY_PENDING); + pending.set(4.0); + gauge!(RELAY_PENDING).increment(2.0); + pending.decrement(3.0); + histogram!(RELAY_CLAIM_DURATION_SECONDS).record(0.005); + let queue_wait_name = "openshell_supervisor_config_queue_wait_seconds"; + histogram!(queue_wait_name).record(0.2); + assert_eq!(handle.is_some(), prometheus); + if let Some(handle) = handle { + let rendered = handle.render(); + let request = rendered.lines().find(|line| line.starts_with("openshell_server_grpc_requests_total{")).unwrap(); + assert!(request.contains("method=\"ListSandboxes\"")); + assert!(request.contains("code=\"0\"")); + assert!(request.ends_with(" 3"), "{request}"); + assert!(rendered.contains("openshell_server_relay_pending 3")); + assert!(rendered.contains("openshell_server_relay_claim_duration_seconds_count 1")); + assert!(rendered.contains(&format!("# TYPE {queue_wait_name} summary"))); + } + for _ in 0..2 { + provider.force_flush().unwrap(); + let batches = exporter.get_finished_metrics().unwrap(); + if !otlp { + assert!(batches.is_empty()); + continue; + } + let metrics: Vec<_> = batches.last().unwrap().scope_metrics() + .flat_map(ScopeMetrics::metrics).collect(); + let counter = metrics.iter().find(|metric| metric.name() == "openshell_server_grpc_requests_total").unwrap(); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = counter.data() else { panic!("expected counter") }; + assert!(sum.is_monotonic()); + assert_eq!(sum.data_points().next().unwrap().value(), 3); + let gauge = metrics.iter().find(|metric| metric.name() == RELAY_PENDING).unwrap(); + assert_eq!(gauge.unit(), "1"); + let AggregatedMetrics::F64(MetricData::Gauge(gauge)) = gauge.data() else { panic!("expected gauge") }; + assert!((gauge.data_points().next().unwrap().value() - 3.0).abs() < f64::EPSILON); + let histogram = metrics.iter().find(|metric| metric.name() == RELAY_CLAIM_DURATION_SECONDS).unwrap(); + assert_eq!(histogram.unit(), "s"); + let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = histogram.data() else { panic!("expected histogram") }; + let point = histogram.data_points().next().unwrap(); + assert_eq!(point.count(), 1); + assert!((point.sum() - 0.005).abs() < f64::EPSILON); + assert_eq!(point.bounds().collect::>(), LATENCY_BUCKETS_SECONDS); + + let queue_wait = metrics + .iter() + .find(|metric| metric.name() == queue_wait_name) + .unwrap(); + let AggregatedMetrics::F64(MetricData::Histogram(queue_wait)) = queue_wait.data() + else { + panic!("expected queue wait histogram"); + }; + let point = queue_wait.data_points().next().unwrap(); + assert_eq!(point.bounds().collect::>(), LATENCY_BUCKETS_SECONDS); + } + provider.shutdown().unwrap(); + }); + } + } + #[test] fn gauge_slot_counts_until_dropped() { let metrics = MetricsCapture::install(); diff --git a/crates/openshell-server/src/gateway_metrics/otlp_recorder.rs b/crates/openshell-server/src/gateway_metrics/otlp_recorder.rs new file mode 100644 index 0000000000..567d027eec --- /dev/null +++ b/crates/openshell-server/src/gateway_metrics/otlp_recorder.rs @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use metrics::{ + Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, Recorder, + SharedString, Unit, +}; +use metrics_util::MetricKind; +use metrics_util::registry::{Registry, Storage}; +use opentelemetry::{KeyValue, metrics as otel}; + +use super::LATENCY_BUCKETS_SECONDS; + +type Descriptions = Mutex>; + +pub struct OtlpRecorder { + instruments: Registry, + descriptions: Arc, +} + +impl OtlpRecorder { + pub(crate) fn new(meter: otel::Meter) -> Self { + let descriptions = Arc::new(Descriptions::default()); + let instruments = Registry::new(Instruments { + meter, + descriptions: Arc::clone(&descriptions), + }); + Self { + instruments, + descriptions, + } + } + + fn describe(&self, kind: MetricKind, name: KeyName, unit: Option, text: SharedString) { + self.descriptions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert((kind, name), (text, unit.map_or("", ucum_unit))); + } +} + +impl Recorder for OtlpRecorder { + fn describe_counter(&self, name: KeyName, unit: Option, description: SharedString) { + self.describe(MetricKind::Counter, name, unit, description); + } + + fn describe_gauge(&self, name: KeyName, unit: Option, description: SharedString) { + self.describe(MetricKind::Gauge, name, unit, description); + } + + fn describe_histogram(&self, name: KeyName, unit: Option, description: SharedString) { + self.describe(MetricKind::Histogram, name, unit, description); + } + + fn register_counter(&self, key: &Key, _: &Metadata<'_>) -> Counter { + self.instruments + .get_or_create_counter(key, |series| Counter::from_arc(Arc::clone(series))) + } + + fn register_gauge(&self, key: &Key, _: &Metadata<'_>) -> Gauge { + self.instruments + .get_or_create_gauge(key, |series| Gauge::from_arc(Arc::clone(series))) + } + + fn register_histogram(&self, key: &Key, _: &Metadata<'_>) -> Histogram { + self.instruments + .get_or_create_histogram(key, |series| Histogram::from_arc(Arc::clone(series))) + } +} + +struct Instruments { + meter: otel::Meter, + descriptions: Arc, +} + +impl Instruments { + fn description(&self, kind: MetricKind, key: &Key) -> (SharedString, &'static str) { + self.descriptions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&(kind, key.name_shared())) + .cloned() + .unwrap_or_default() + } +} + +impl Storage for Instruments { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, key: &Key) -> Self::Counter { + let (description, unit) = self.description(MetricKind::Counter, key); + let instrument = self + .meter + .u64_counter(key.name_shared().into_inner()) + .with_description(description) + .with_unit(unit) + .build(); + let attributes = attributes(key); + instrument.add(0, &attributes); + Arc::new(CounterSeries { + instrument, + attributes, + total: AtomicU64::new(0), + }) + } + + fn gauge(&self, key: &Key) -> Self::Gauge { + let (description, unit) = self.description(MetricKind::Gauge, key); + let instrument = self + .meter + .f64_gauge(key.name_shared().into_inner()) + .with_description(description) + .with_unit(unit) + .build(); + let attributes = attributes(key); + instrument.record(0.0, &attributes); + Arc::new(GaugeSeries { + instrument, + attributes, + value: Mutex::new(0.0), + }) + } + + fn histogram(&self, key: &Key) -> Self::Histogram { + let (description, unit) = self.description(MetricKind::Histogram, key); + let mut builder = self + .meter + .f64_histogram(key.name_shared().into_inner()) + .with_description(description) + .with_unit(unit); + if key.name().ends_with("_seconds") { + builder = builder.with_boundaries(LATENCY_BUCKETS_SECONDS.to_vec()); + } + Arc::new(HistogramSeries { + instrument: builder.build(), + attributes: attributes(key), + }) + } +} + +fn attributes(key: &Key) -> Vec { + key.labels() + .map(|label| KeyValue::new(label.key().to_owned(), label.value().to_owned())) + .collect() +} + +fn ucum_unit(unit: Unit) -> &'static str { + match unit { + Unit::Count => "1", + Unit::Microseconds => "us", + Unit::Bytes => "By", + Unit::Kibibytes => "KiBy", + Unit::Mebibytes => "MiBy", + Unit::Gibibytes => "GiBy", + Unit::Tebibytes => "TiBy", + Unit::BitsPerSecond => "bit/s", + Unit::KilobitsPerSecond => "kbit/s", + Unit::MegabitsPerSecond => "Mbit/s", + Unit::GigabitsPerSecond => "Gbit/s", + Unit::TerabitsPerSecond => "Tbit/s", + Unit::CountPerSecond => "1/s", + _ => unit.as_canonical_label(), + } +} + +struct CounterSeries { + instrument: otel::Counter, + attributes: Vec, + total: AtomicU64, +} + +impl CounterFn for CounterSeries { + fn increment(&self, value: u64) { + self.total.fetch_add(value, Ordering::Relaxed); + self.instrument.add(value, &self.attributes); + } + + fn absolute(&self, value: u64) { + let previous = self.total.fetch_max(value, Ordering::Relaxed); + self.instrument + .add(value.saturating_sub(previous), &self.attributes); + } +} + +struct GaugeSeries { + instrument: otel::Gauge, + attributes: Vec, + value: Mutex, +} + +impl GaugeSeries { + fn update(&self, change: impl FnOnce(f64) -> f64) { + // A delayed recording could overwrite a newer sample if it ran outside this lock. + let mut value = self.value.lock().unwrap_or_else(PoisonError::into_inner); + *value = change(*value); + self.instrument.record(*value, &self.attributes); + } +} + +impl GaugeFn for GaugeSeries { + fn increment(&self, delta: f64) { + self.update(|value| value + delta); + } + + fn decrement(&self, delta: f64) { + self.update(|value| value - delta); + } + + fn set(&self, value: f64) { + self.update(|_| value); + } +} + +struct HistogramSeries { + instrument: otel::Histogram, + attributes: Vec, +} + +impl HistogramFn for HistogramSeries { + fn record(&self, value: f64) { + self.instrument.record(value, &self.attributes); + } +} + +#[cfg(test)] +mod tests { + use std::sync::{TryLockError, mpsc}; + use std::time::Duration; + + use super::*; + + const WAIT: Duration = Duration::from_secs(5); + + struct PausedMeasurement { + entered: mpsc::Sender<()>, + resume: Mutex>, + } + + impl otel::SyncInstrument for PausedMeasurement { + fn measure(&self, _: f64, _: &[KeyValue]) { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv_timeout(WAIT).unwrap(); + } + } + + #[test] + fn gauge_update_holds_the_lock_during_recording() { + let (entered, recording) = mpsc::channel(); + let (release, resume) = mpsc::channel(); + let gauge = GaugeSeries { + instrument: otel::Gauge::new(Arc::new(PausedMeasurement { + entered, + resume: Mutex::new(resume), + })), + attributes: Vec::new(), + value: Mutex::new(0.0), + }; + + std::thread::scope(|scope| { + scope.spawn(|| gauge.increment(1.0)); + recording.recv_timeout(WAIT).unwrap(); + let update_blocked = matches!(gauge.value.try_lock(), Err(TryLockError::WouldBlock)); + release.send(()).unwrap(); + assert!( + update_blocked, + "another update could overtake this recording" + ); + }); + } +} diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 4f591f004f..c0ae6f0649 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -759,6 +759,7 @@ pub(crate) async fn run_server( startup: ServerStartupConfig, tracing_log_bus: TracingLogBus, compute_drivers: ComputeDriverRegistry, + metrics_recorder: Option, ) -> Result<()> { let ServerStartupConfig { config, @@ -1129,17 +1130,14 @@ pub(crate) async fn run_server( // Install the recorder before the endpoint-status startup reconciliation so // the lock waits and timeouts recorded there are exported. Its listener // binds with the others. - let metrics_endpoint = config - .metrics_bind_address - .map(|metrics_bind_address| { - gateway_metrics::install_global_recorder( - supervisor_session::RELAY_CAPACITY, - store.mutation_lock_connections(), - ) - .map(|prometheus_handle| (metrics_bind_address, prometheus_handle)) - }) - .transpose() - .map_err(|e| Error::config(format!("failed to install metrics recorder: {e}")))?; + let prometheus_handle = gateway_metrics::install_global_recorder( + config.metrics_bind_address.is_some(), + metrics_recorder, + supervisor_session::RELAY_CAPACITY, + store.mutation_lock_connections(), + ) + .map_err(|e| Error::config(format!("failed to install metrics recorder: {e}")))?; + let metrics_endpoint = config.metrics_bind_address.zip(prometheus_handle); grpc::policy::backfill_legacy_policy_history(&state) .await diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index ca799ba411..301c052d98 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -11,29 +11,33 @@ //! //! `[openshell.gateway.otlp]` decides **whether and where** to export each //! signal: a shared `endpoint` enables traces, `[otlp.traces]` overrides -//! their endpoint, and `[otlp.ocsf]` with `enabled = true` exports OCSF -//! records, optionally to their own endpoint. `OTEL_EXPORTER_OTLP_*ENDPOINT` is deliberately not -//! read, so enablement has one source. +//! their endpoint, `[otlp.metrics]` with `enabled = true` exports gateway +//! metrics, and `[otlp.ocsf]` with `enabled = true` exports OCSF records. +//! Each signal can use its own endpoint. `OTEL_EXPORTER_OTLP_*ENDPOINT` is +//! deliberately not read, so enablement has one source. //! //! **How** to export — sampling, batching, span limits, transport headers — //! is the SDK's `OTEL_*` environment surface, read as the provider is built //! and mirrored nowhere here. `docs/how-it-works/gateways/configuration.mdx` documents //! the variables operators are likely to want. //! -//! Diagnostic logs and metrics are not exported over OTLP. They have their own -//! surfaces (the gateway log stream and the Prometheus `/metrics` endpoint). +//! Gateway metrics are exported when `[otlp.metrics]` sets `enabled = true`. +//! Diagnostic logs remain in the gateway log stream. use openshell_core::jwt::{OtlpTlsMaterial, SecretOtlpHeaders}; use openshell_core::sandbox_env::OtlpProtocol; use openshell_otel::{OcsfOtlp, OtlpExportConfig, OtlpTarget, ServiceName}; pub use openshell_otel::{SetupError, TraceContextInterceptor, mark_error}; +use opentelemetry::metrics::MeterProvider as _; #[cfg(test)] use opentelemetry_sdk::Resource; +use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::trace::SdkTracerProvider; use tracing::Subscriber; use tracing_subscriber::registry::LookupSpan; use crate::config_file::{OtlpCollectorConfig, OtlpConfig}; +use crate::gateway_metrics::OtlpRecorder; /// `service.name` reported when the config file does not override it. const DEFAULT_SERVICE_NAME: &str = "openshell-gateway"; @@ -114,11 +118,11 @@ fn build_provider( openshell_otel::build_provider(&export_config(cfg, endpoint, gateway)) } -/// OTLP export configured by `[openshell.gateway.otlp]`: gateway spans and -/// OCSF log records, each sent to its own configured collector. +/// OTLP export configured by `[openshell.gateway.otlp]`. pub struct OtlpExport { traces: Option>, ocsf: Option>, + metrics: Option>, /// Validated collector that supervisors relay workload traces to. The /// gateway itself exports nothing for this signal. agent: Option>, @@ -203,6 +207,7 @@ impl OtlpExport { Ok(Self { traces, ocsf, + metrics: None, agent, traces_transport: cfg .traces @@ -212,6 +217,47 @@ impl OtlpExport { }) } + /// Start gateway metrics independently so failure leaves other signals enabled. + pub fn start_metrics( + &mut self, + cfg: &OtlpConfig, + gateway: GatewayResourceAttributes<'_>, + ) -> Result, SetupError> { + let Some(collector) = &cfg.metrics else { + return Ok(None); + }; + let mut config = export_config(cfg, &collector.endpoint, gateway); + if openshell_otel::resource_for(&config) + .get(&opentelemetry::Key::from_static_str("service.instance.id")) + .is_none() + { + config + .resource_attributes + .push(opentelemetry::KeyValue::new( + "service.instance.id", + uuid::Uuid::new_v4().to_string(), + )); + } + let target = + OtlpTarget::new(&config)?.with_transport(&openshell_otel::CollectorTransport { + protocol: collector.protocol.into(), + headers: collector.headers.header_map(), + tls: collector.tls.clone(), + }); + let provider = target.meter_provider()?; + let recorder = OtlpRecorder::new(provider.meter(INSTRUMENTATION_SCOPE)); + self.metrics = Some(Signal { + endpoint: target.endpoint().to_string(), + protocol: collector.protocol, + exporter: provider, + }); + Ok(Some(recorder)) + } + + pub fn metrics_endpoint(&self) -> Option<&str> { + self.metrics.as_ref().map(|signal| signal.endpoint.as_str()) + } + /// The collector that supervisors relay workload traces to, when the /// agent trace relay is enabled. pub fn agent_endpoint(&self) -> Option<&str> { @@ -262,9 +308,15 @@ impl OtlpExport { self.ocsf.as_ref().map(|ocsf| &ocsf.exporter) } - /// Flush buffered spans, then export queued OCSF records within their - /// shutdown budget. + /// Flush gateway metrics, spans, and queued OCSF records. pub async fn shutdown(self) { + if let Some(metrics) = self.metrics { + match tokio::task::spawn_blocking(move || metrics.exporter.shutdown()).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!(%error, "OTLP meter provider shutdown failed"), + Err(error) => tracing::warn!(%error, "OTLP meter provider shutdown task failed"), + } + } if let Some(traces) = self.traces && let Err(err) = traces.exporter.shutdown() { @@ -760,6 +812,7 @@ mod tests { fn with_traces_endpoint(endpoint: &str) -> OtlpConfig { OtlpConfig { + metrics: None, service_name: None, traces: Some(OtlpTracesConfig { collector: OtlpCollectorConfig::new(endpoint, OtlpProtocol::Grpc), @@ -929,6 +982,7 @@ mod tests { OtlpCollectorConfig::new(endpoint, OtlpProtocol::Grpc).with_tls(mutual()) }; let otlp = OtlpConfig { + metrics: None, service_name: None, traces: Some(OtlpTracesConfig { collector: collector("https://shared:4317"), @@ -976,6 +1030,75 @@ mod tests { export.shutdown().await; } + #[tokio::test] + async fn metrics_have_distinct_process_resources_and_flush_on_shutdown() { + Environment::new() + .remove("OTEL_RESOURCE_ATTRIBUTES") + .remove("OTEL_SERVICE_NAME") + .run_async(async { + let collector = openshell_otel_test_support::OtlpTestServer::start().await; + let mut cfg = with_traces_endpoint(collector.endpoint()); + cfg.metrics = Some(OtlpCollectorConfig::new( + collector.endpoint(), + OtlpProtocol::Grpc, + )); + for _ in 0..2 { + let (export, error) = + OtlpExport::start(Some(&cfg), GatewayResourceAttributes::default()); + assert!(error.is_none()); + let mut export = export.unwrap(); + let recorder = export + .start_metrics( + &cfg, + GatewayResourceAttributes::new(Some("production"), Some("docker")), + ) + .unwrap() + .unwrap(); + metrics::with_local_recorder(&recorder, || { + metrics::counter!("resource_test_total").increment(1); + }); + export.shutdown().await; + } + let received = collector.shutdown().await; + assert_eq!(received.metric_exports.len(), 2); + let resources = received.resource_attributes; + assert_eq!(resources[0]["openshell.gateway.name"], "production"); + assert_eq!(resources[0]["openshell.gateway.compute_driver"], "docker"); + assert_eq!(resources[0]["service.name"], "openshell-gateway"); + assert_ne!( + resources[0]["service.instance.id"], + resources[1]["service.instance.id"] + ); + }) + .await; + } + + #[tokio::test] + async fn metrics_setup_failure_preserves_traces() { + let mut cfg = config(); + let (export, error) = OtlpExport::start(Some(&cfg), GatewayResourceAttributes::default()); + assert!(error.is_none()); + let mut export = export.unwrap(); + assert!( + export + .start_metrics(&cfg, GatewayResourceAttributes::default()) + .unwrap() + .is_none() + ); + cfg.metrics = Some(OtlpCollectorConfig::new( + "not a valid URI", + OtlpProtocol::Grpc, + )); + assert!( + export + .start_metrics(&cfg, GatewayResourceAttributes::default()) + .is_err() + ); + assert!(export.tracer_provider().is_some()); + assert!(export.metrics_endpoint().is_none()); + export.shutdown().await; + } + #[tokio::test] async fn ocsf_records_reach_an_otlp_collector_over_grpc() { use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, ocsf_emit}; diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index cec79ad5a7..3c742da53a 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -799,7 +799,7 @@ Allow enough retention and disk space for shipper outages. Graceful shutdown all ## OTLP Export -`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP. Omit the table to disable export. Its `endpoint` is the collector for every signal the gateway exports and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. [OCSF log export](#ocsf-log-export) is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and the [agent trace relay](#agent-trace-relay) is off until `[openshell.gateway.otlp.agent]` sets `enabled = true`. Each can use its own collector. [OTLP Collectors](/observability/overview#otlp-collectors) shows where each signal goes. +`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP. Omit the table to disable export. Its `endpoint` is the default collector for each signal and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. OTLP metrics are off until `[openshell.gateway.otlp.metrics]` sets `enabled = true`, OCSF log export is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and the [agent trace relay](#agent-trace-relay) is off until `[openshell.gateway.otlp.agent]` sets `enabled = true`. Each signal can use its own collector. [OTLP Collectors](/observability/overview#otlp-collectors) shows where each signal goes. The gateway already uses the Rust `tracing` framework for structured logs sent to stdout and the sandbox log stream. Enabling this section adds an OpenTelemetry layer to the same tracing subscriber. It exports span trees to an OTLP collector without exporting, replacing, or redirecting the existing diagnostic log events. @@ -816,9 +816,9 @@ protocol = "http/protobuf" Each enabled signal needs an endpoint, either in its own table or from `endpoint`. A table without its own `endpoint` uses the whole `[openshell.gateway.otlp]` collector, including its `protocol`, headers, and TLS files. A table with its own `endpoint` takes nothing else from `[openshell.gateway.otlp]`, so configured settings for one collector, such as its credentials, never reach another, and setting any of them in a table without an `endpoint` is a configuration error. Endpoints must be valid URIs without a query string: exporters send no query parameters, so an endpoint with one is a configuration error, and a collector that needs parameters must take them as headers. If an endpoint is malformed, the gateway logs the configuration error and continues with all OTLP export disabled. It does not connect at startup: an unreachable collector produces export failures, never a failure to serve. -`protocol` selects the transport to a collector: `grpc`, the default, or `http/protobuf`. With `http/protobuf`, the endpoint is a base URL and each signal is posted to its path under it, `/v1/traces` or `/v1/logs`, as with `OTEL_EXPORTER_OTLP_ENDPOINT`. HTTP/JSON is not supported. Point a `grpc` endpoint at a collector's gRPC receiver, conventionally port `4317`, and an `http/protobuf` endpoint at its HTTP receiver, conventionally port `4318`. A URI alone cannot distinguish the two, so a mismatched endpoint is accepted at startup and then fails on export. Sandbox supervisors and standalone compute drivers must come from the same release as the gateway to use `http/protobuf`: an older supervisor exports over gRPC, and an older driver rejects the protocol setting. +`protocol` selects the transport to a collector: `grpc`, the default, or `http/protobuf`. With `http/protobuf`, the endpoint is a base URL and each signal is posted to its path under it, `/v1/traces`, `/v1/logs`, or `/v1/metrics`, as with `OTEL_EXPORTER_OTLP_ENDPOINT`. HTTP/JSON is not supported. Point a `grpc` endpoint at a collector's gRPC receiver, conventionally port `4317`, and an `http/protobuf` endpoint at its HTTP receiver, conventionally port `4318`. A URI alone cannot distinguish the two, so a mismatched endpoint is accepted at startup and then fails on export. Sandbox supervisors and standalone compute drivers must come from the same release as the gateway to use `http/protobuf`: an older supervisor exports over gRPC, and an older driver rejects the protocol setting. -`headers` sets request headers sent with every export to a collector, as gRPC metadata or HTTP headers, such as an authorization token or a backend's routing header. Each name must be a valid HTTP header name set once, ignoring case; names the exporters set themselves, such as `content-type`, `host`, and `grpc-*`, and binary gRPC names ending in `-bin` are rejected. Each value must be non-empty printable ASCII; spaces are allowed. The gateway's trace and OCSF exporters also read `OTEL_EXPORTER_OTLP_TRACES_HEADERS` or `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, or else `OTEL_EXPORTER_OTLP_HEADERS`, and a header set there replaces a configured header of the same name. In-process compute drivers and the VM driver send the gateway's trace headers, including those from the environment, with their own spans. A standalone compute driver sends only the headers in its own `OTEL_EXPORTER_OTLP_*HEADERS` variables. Supervisors send their trace collector's `headers` with their own spans, and the gateway's OCSF headers, including those from the environment, with their OCSF records. Trace headers from the gateway's environment do not reach supervisors, so set `headers` or `headers_file` on the trace collector when supervisor spans need them. The agent trace relay sends only its collector's `headers`, never ones from the environment, so its credentials can only be set with `headers` or `headers_file`. A changed header takes effect when the gateway restarts, and reaches a running sandbox only after that sandbox restarts. +`headers` sets request headers sent with every export to a collector, as gRPC metadata or HTTP headers, such as an authorization token or a backend's routing header. Each name must be a valid HTTP header name set once, ignoring case; names the exporters set themselves, such as `content-type`, `host`, and `grpc-*`, and binary gRPC names ending in `-bin` are rejected. Each value must be non-empty printable ASCII; spaces are allowed. The gateway's exporters also read their signal's `OTEL_EXPORTER_OTLP_TRACES_HEADERS`, `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, or `OTEL_EXPORTER_OTLP_METRICS_HEADERS`, or else `OTEL_EXPORTER_OTLP_HEADERS`, and a header set there replaces a configured header of the same name. In-process compute drivers and the VM driver send the gateway's trace headers, including those from the environment, with their own spans. A standalone compute driver sends only the headers in its own `OTEL_EXPORTER_OTLP_*HEADERS` variables. Supervisors send their trace collector's `headers` with their own spans, and the gateway's OCSF headers, including those from the environment, with their OCSF records. Trace headers from the gateway's environment do not reach supervisors, so set `headers` or `headers_file` on the trace collector when supervisor spans need them. The agent trace relay sends only its collector's `headers`, never ones from the environment, so its credentials can only be set with `headers` or `headers_file`. A changed header takes effect when the gateway restarts, and reaches a running sandbox only after that sandbox restarts. Put credentials in `headers_file` rather than `headers`. It reads a collector's headers from a file instead, such as a mounted Kubernetes Secret, so credentials stay out of the configuration file. The file holds one `name=value` header per line. Blank lines are skipped, and whitespace around each name and value is trimmed. The headers are validated like `headers`, the file must be UTF-8 and at most 64 KiB, and setting both `headers` and `headers_file` in one table is a configuration error. A relative path uses the gateway's working directory. The gateway reads the file at startup and fails to start if it is missing or invalid, so a changed file takes effect as a changed header does. @@ -925,6 +925,32 @@ automatically. In-process compute drivers read their backend-specific settings from `[openshell.drivers.]`. An external driver's gateway table supplies only its `socket_path`; configure the external driver process itself through that binary's flags or environment variables. A driver that advertises `supports_sandbox_authentication` may authenticate an opaque bootstrap credential through the compute-driver protocol. It must return the same non-empty runtime identity from sandbox creation and credential authentication. Start requests include the previously recorded opaque identity so the driver can preserve the stable resource while rotating generation-specific compute. For `IssueSandboxToken`, the gateway verifies the sandbox exists and the driver/runtime identity matches the durable provisioning record before returning a generation-bound session JWT. The in-process Kubernetes driver reads `service_account_name`, `workspace_mode`, and namespace discovery from `[openshell.drivers.kubernetes]`; an external Kubernetes driver receives the equivalent values through its own CLI or environment contract. +### OTLP Metrics + +Set `enabled = true` in `[openshell.gateway.otlp.metrics]` to push gateway +metrics over OTLP. It defaults to `false` and uses the table's collector +settings, or inherits the shared endpoint, protocol, headers, and TLS settings +from `[openshell.gateway.otlp]` when its own `endpoint` is omitted. This works +even when `[openshell.gateway.otlp.traces]` selects a different collector. + +```toml +[openshell.gateway.otlp.metrics] +enabled = true +# endpoint = "https://metrics-collector:4317" # optional override +``` + +Metric export works with or without the Prometheus listener. Metric names and +labels are preserved: for example, `openshell_server_grpc_requests_total` +keeps that name, and `openshell_server_grpc_request_duration_seconds` is one +OTLP histogram with that name, not separate `_bucket`, `_sum`, or `_count` +metric names. Prometheus summary quantiles are not exported. Metrics carry the +gateway's trace resource attributes and a `service.instance.id` that distinguishes +gateway processes. Set it through `OTEL_RESOURCE_ATTRIBUTES` to override the +generated process identity. Supervisors and workloads do not export metrics +through this setting. + +For Helm, set `gatewayConfig."openshell.gateway.otlp.metrics".enabled: true`. + ### OCSF Log Export Set `enabled = true` in `[openshell.gateway.otlp.ocsf]` to export gateway OCSF events as OTLP log records. It defaults to `false`, so trace export alone never sends OCSF events. Records go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, with the same resource attributes as spans, under the `openshell.ocsf` instrumentation scope. Use a separate endpoint to send OCSF events to a collector that feeds a SIEM while spans go elsewhere. Sandbox supervisors receive the same endpoint and protocol and export their own OCSF events there directly, so the endpoint must also be reachable from sandboxes. Supervisor records carry the supervisor's resource attributes, including `openshell.sandbox.id` and `openshell.gateway.name`. Supervisors also send the collector [headers](#otlp-export) that the gateway uses for OCSF export, so a collector that requires authentication accepts their records too. The gateway delivers headers with each sandbox's launch credentials, never through the sandbox environment. Other `OTEL_*` variables do not reach supervisors. @@ -992,6 +1018,8 @@ The gateway honors these variables: | `OTEL_EXPORTER_OTLP_COMPRESSION` | Set to `gzip` to compress exports of every signal. `zstd` is not supported. | | `OTEL_EXPORTER_OTLP_TRACES_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION`, `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` | Exporter transport tuning for spans only. These override the general variables. | | `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, `OTEL_EXPORTER_OTLP_LOGS_COMPRESSION`, `OTEL_EXPORTER_OTLP_LOGS_TIMEOUT` | Exporter transport tuning for OCSF log records only. These override the general variables. | +| `OTEL_METRIC_EXPORT_INTERVAL` | Interval between gateway metric exports in milliseconds. Defaults to `60000`. | +| `OTEL_EXPORTER_OTLP_METRICS_HEADERS`, `OTEL_EXPORTER_OTLP_METRICS_COMPRESSION`, `OTEL_EXPORTER_OTLP_METRICS_TIMEOUT` | Metric exporter transport tuning. These override the general variables. | | `SSL_CERT_FILE`, `SSL_CERT_DIR` | Replace the system trust roots for `https://` endpoints without a `ca_file`. They affect the gateway and the compute drivers it runs, not supervisors. | For Helm deployments, set these variables with `server.extraEnv`. Read credentials from a Secret so they stay out of the values file: @@ -1013,13 +1041,12 @@ These variables have no effect. Only `[openshell.gateway.otlp]` decides which si | Variable | Reason | |---|---| -| `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Enablement has one source, `[openshell.gateway.otlp]`, so an environment variable cannot silently turn export on or redirect it. | +| `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Enablement has one source, `[openshell.gateway.otlp]`, so an environment variable cannot silently turn export on or redirect it. | | `OTEL_SDK_DISABLED`, `OTEL_TRACES_EXPORTER`, `OTEL_LOGS_EXPORTER`, `OTEL_METRICS_EXPORTER` | `[openshell.gateway.otlp]` decides which signals are exported. To disable export, omit it. | -| `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | `protocol` in each table selects the transport, and supervisors follow the same setting. | +| `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL`, `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL` | `protocol` in each table selects the transport, and supervisors follow the same setting. | | `OTEL_EXPORTER_OTLP_INSECURE`, `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`, and their per-signal forms | The endpoint scheme selects TLS, and `ca_file`, `cert_file`, and `key_file` in each table set its trust and client certificate, so supervisors follow the same setting. | | `OTEL_PROPAGATORS` | The gateway always propagates W3C Trace Context. | | `OTEL_BLRP_SCHEDULE_DELAY`, `OTEL_BLRP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BLRP_EXPORT_TIMEOUT` | OCSF log export uses its own queue and batching, described in [OCSF Log Export](#ocsf-log-export). | -| `OTEL_EXPORTER_OTLP_METRICS_*`, `OTEL_METRIC_EXPORT_INTERVAL` | The gateway serves metrics for Prometheus scraping and does not export them over OTLP. | To sample 10% of traces: diff --git a/docs/observability/gateway-metrics.mdx b/docs/observability/gateway-metrics.mdx index 43c659bdbf..5f24a9e192 100644 --- a/docs/observability/gateway-metrics.mdx +++ b/docs/observability/gateway-metrics.mdx @@ -15,8 +15,10 @@ to alert on saturation, to follow rollouts, and to choose autoscaling signals. ## Enable the Metrics Endpoint The gateway serves `GET /metrics` in the Prometheus text format on its own -listener, separate from the API and health ports. When the listener is -disabled, the gateway records no metrics. Enable it in one of these ways: +listener, separate from the API and health ports. Metrics can also be pushed +over OTLP independently of this listener; refer to +[OTLP Metrics](/how-it-works/gateways/configuration#otlp-metrics). +Enable the scrape listener in one of these ways: - In `gateway.toml`, set `metrics_bind_address` under `[openshell.gateway]`, for example `metrics_bind_address = "0.0.0.0:9090"`. Refer to diff --git a/docs/observability/overview.mdx b/docs/observability/overview.mdx index 94194d805c..d6b5cfe6bd 100644 --- a/docs/observability/overview.mdx +++ b/docs/observability/overview.mdx @@ -14,7 +14,7 @@ OpenShell produces three kinds of observability signals: traces, metrics, and OC |---|---|---| | Traces | Gateway, compute drivers, sandbox supervisors | OTLP collector | | Agent traces | Agents, relayed by sandbox supervisors | OTLP collector | -| Metrics | Gateway | Prometheus scrape | +| Metrics | Gateway | Prometheus scrape, optional OTLP push | | OCSF events, shorthand | Gateway, sandbox supervisors | Gateway console, `openshell logs`, supervisor log file | | OCSF events, JSONL | Gateway, sandbox supervisors | Local file | | OCSF events, OTLP | Gateway, sandbox supervisors | OTLP collector | @@ -68,6 +68,9 @@ While the relay is on, every workload process gets `OTEL_EXPORTER_OTLP_ENDPOINT` The gateway serves Prometheus metrics for API requests, readiness, gateway interceptors, OCSF output, and multi-replica capacity, on a listener separate from its API. Use them to alert on saturation and to choose autoscaling signals. +To push these metrics to an OTLP collector independently of the scrape +listener, refer to [OTLP Metrics](/how-it-works/gateways/configuration#otlp-metrics). + To enable the listener, set `metrics_bind_address` in `gateway.toml`: ```toml diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 578097b5ad..c1532980cc 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1628,6 +1628,7 @@ credential failures. | Symptom | Likely cause | Check | |---|---|---| +| Collector receives no gateway metrics | OTLP metric export is off, or its collector cannot accept metrics | Set `enabled = true` in `[openshell.gateway.otlp.metrics]`, inspect `metrics_endpoint` in the gateway's `OTLP exporting enabled` log, and check exporter errors. Metrics use their table's collector, or inherit the shared collector when no endpoint is set; the Prometheus listener is configured independently. See [OTLP metrics](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration#otlp-metrics). | | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable endpoint or one that does not speak the table's `protocol`, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint` and `ocsf_protocol`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, or shipper falling behind retention | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). |