Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ tracing-opentelemetry = { version = "0.34", default-features = false, features =

# Metrics
metrics = "0.24"
metrics-util = { version = "0.20", default-features = false }
metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] }

# Unix/Process
Expand Down
5 changes: 5 additions & 0 deletions crates/openshell-gateway/src/vm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -942,6 +942,7 @@ mod tests {
.unwrap();
assert!(collector.tls().client_identity().is_some());
let otlp = OtlpConfig {
metrics: None,
service_name: None,
traces: Some(OtlpTracesConfig { collector }),
ocsf: None,
Expand Down Expand Up @@ -974,6 +975,7 @@ mod tests {
append_otlp_args(
&mut command,
Some(&OtlpConfig {
metrics: None,
service_name: Some("custom-gateway".to_string()),
traces: Some(OtlpTracesConfig {
collector: OtlpCollectorConfig::new(
Expand Down Expand Up @@ -1006,6 +1008,7 @@ mod tests {
append_otlp_args(
&mut command,
Some(&OtlpConfig {
metrics: None,
service_name: None,
traces: Some(OtlpTracesConfig {
collector: OtlpCollectorConfig::new(
Expand Down Expand Up @@ -1040,6 +1043,7 @@ mod tests {
append_otlp_args(
&mut command,
Some(&OtlpConfig {
metrics: None,
service_name: None,
traces: None,
ocsf: Some(OcsfOtlpConfig {
Expand Down Expand Up @@ -1078,6 +1082,7 @@ mod tests {
append_otlp_args(
&mut command,
Some(&OtlpConfig {
metrics: None,
service_name: None,
traces: None,
ocsf: None,
Expand Down
2 changes: 1 addition & 1 deletion crates/openshell-otel-test-support/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ hyper = { workspace = true }
hyper-util = { workspace = true, features = ["tokio"] }
openshell-crypto = { workspace = true }
openshell-otel = { path = "../openshell-otel" }
opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace"] }
opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace", "metrics"] }
prost = { workspace = true }
rcgen = { workspace = true }
rustls = { workspace = true }
Expand Down
45 changes: 44 additions & 1 deletion crates/openshell-otel-test-support/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ use opentelemetry_proto::tonic::collector::logs::v1::{
ExportLogsServiceRequest, ExportLogsServiceResponse,
logs_service_server::{LogsService, LogsServiceServer},
};
use opentelemetry_proto::tonic::collector::metrics::v1::{
ExportMetricsServiceRequest, ExportMetricsServiceResponse,
metrics_service_server::{MetricsService, MetricsServiceServer},
};
use opentelemetry_proto::tonic::collector::trace::v1::{
ExportTraceServiceRequest, ExportTraceServiceResponse,
trace_service_server::{TraceService, TraceServiceServer},
Expand Down Expand Up @@ -141,6 +145,10 @@ pub async fn assert_compute_driver_tracing(

#[derive(Clone, Debug, Default)]
pub struct ReceivedTraces {
/// Raw metric export requests, including resources and aggregation data.
pub metric_exports: Vec<ExportMetricsServiceRequest>,
/// `authorization` metadata of each metric export request.
pub metric_authorizations: Vec<String>,
pub spans: Vec<Span>,
/// String bodies of exported log records.
pub log_bodies: Vec<String>,
Expand Down Expand Up @@ -281,6 +289,38 @@ impl LogsService for Collector {
}
}

#[tonic::async_trait]
impl MetricsService for Collector {
async fn export(
&self,
request: tonic::Request<ExportMetricsServiceRequest>,
) -> Result<tonic::Response<ExportMetricsServiceResponse>, tonic::Status> {
{
let mut received = self
.received
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(authorization) = request
.metadata()
.get("authorization")
.and_then(|value| value.to_str().ok())
{
received
.metric_authorizations
.push(authorization.to_string());
}
let request = request.into_inner();
for metrics in &request.resource_metrics {
let attributes = received.record_resource(metrics.resource.clone());
received.resource_attributes.push(attributes);
}
received.metric_exports.push(request);
}
self.exported.notify_one();
Ok(tonic::Response::new(ExportMetricsServiceResponse::default()))
}
}

/// Loopback OTLP/gRPC server that captures exported spans, logs, and resources.
pub struct OtlpTestServer {
endpoint: String,
Expand Down Expand Up @@ -353,13 +393,16 @@ impl OtlpTestServer {
}
let trace_service = TraceServiceServer::new(collector.clone())
.accept_compressed(tonic::codec::CompressionEncoding::Gzip);
let logs_service = LogsServiceServer::new(collector)
let logs_service = LogsServiceServer::new(collector.clone())
.accept_compressed(tonic::codec::CompressionEncoding::Gzip);
let metrics_service = MetricsServiceServer::new(collector)
.accept_compressed(tonic::codec::CompressionEncoding::Gzip);
let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
server
.add_service(trace_service)
.add_service(logs_service)
.add_service(metrics_service)
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
async {
Expand Down
6 changes: 3 additions & 3 deletions crates/openshell-otel/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

[package]
name = "openshell-otel"
description = "Shared OpenTelemetry trace and log export support for OpenShell services"
description = "Shared OpenTelemetry export support for OpenShell services"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
Expand All @@ -24,7 +24,7 @@ openshell-ocsf = { path = "../openshell-ocsf", optional = true }
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true }
opentelemetry-http = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true, features = ["gzip-http", "http-proto", "logs"] }
opentelemetry-otlp = { workspace = true, optional = true, features = ["gzip-http", "http-proto", "logs", "metrics"] }
rustls = { workspace = true, optional = true }
rustls-native-certs = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
Expand All @@ -39,7 +39,7 @@ zeroize = { workspace = true, optional = true }

[dev-dependencies]
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
opentelemetry-proto = { workspace = true, features = ["logs"] }
opentelemetry-proto = { workspace = true, features = ["logs", "metrics"] }
opentelemetry_sdk = { workspace = true, features = ["testing"] }
prost = { workspace = true }
tempfile = "3"
Expand Down
6 changes: 4 additions & 2 deletions crates/openshell-otel/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Shared OpenTelemetry trace and log export support for `OpenShell` services.
//! Shared OpenTelemetry export support for `OpenShell` services.
//!
//! The crate has two tiers:
//!
//! - Trace-context propagation, always compiled in: W3C `traceparent` /
//! `tracestate` extraction and injection for HTTP headers, tonic metadata,
//! and process environments.
//! - The `export` feature (on by default): OTLP tracer providers, tracing
//! - The `export` feature (on by default): OTLP tracer and meter providers, tracing
//! layers, compute-driver and gRPC tracing machinery, and the OCSF log
//! bridge. Services that report to a collector keep the default; clients
//! that only forward trace context (e.g. the CLI) opt out with
Expand All @@ -35,6 +35,8 @@ mod headers;
#[cfg(feature = "export")]
mod logs;
#[cfg(feature = "export")]
mod metrics;
#[cfg(feature = "export")]
mod ocsf;
#[cfg(feature = "export")]
mod setup;
Expand Down
173 changes: 173 additions & 0 deletions crates/openshell-otel/src/metrics.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! OTLP metric export support.

use opentelemetry_otlp::MetricExporter;
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};

use crate::{OtlpTarget, Protocol, SetupError};

impl OtlpTarget {
/// Build a meter provider that periodically exports to this collector.
pub fn meter_provider(&self) -> Result<SdkMeterProvider, SetupError> {
self.check_tls()?;
let exporter = match self.protocol {
Protocol::Grpc => self
.configure(MetricExporter::builder().with_tonic())
.build()?,
Protocol::HttpProtobuf => self
.configure_http(
MetricExporter::builder().with_http(),
"v1/metrics",
"OTEL_EXPORTER_OTLP_METRICS_TIMEOUT",
)?
.build()?,
};
Ok(SdkMeterProvider::builder()
.with_reader(PeriodicReader::builder(exporter).build())
.with_resource(self.resource().clone())
.build())
}
}

#[cfg(test)]
mod tests {
use openshell_otel_test_support::{OtlpHttpTestServer, OtlpTestServer, ServerTls, TestPki};
use opentelemetry::{KeyValue, metrics::MeterProvider as _};
use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
use opentelemetry_proto::tonic::metrics::v1::{metric, number_data_point};
use prost::Message as _;

use super::*;
use crate::{OtlpExportConfig, OtlpTls, ServiceName};

fn target(endpoint: &str, protocol: Protocol) -> OtlpTarget {
OtlpTarget::new(&OtlpExportConfig {
endpoint,
service_name: ServiceName::Fixed("metrics-test"),
service_version: Some("1.2.3"),
resource_attributes: vec![KeyValue::new("openshell.gateway.name", "test-gateway")],
})
.unwrap()
.with_protocol(protocol)
}

fn assert_counter(request: &ExportMetricsServiceRequest) {
let resource = &request.resource_metrics[0];
assert!(
resource
.resource
.as_ref()
.unwrap()
.attributes
.iter()
.any(|attribute| attribute.key == "service.name"
&& format!("{:?}", attribute.value).contains("metrics-test"))
);
let metric = &resource.scope_metrics[0].metrics[0];
assert_eq!(metric.name, "requests_total");
let Some(metric::Data::Sum(sum)) = &metric.data else {
panic!("expected counter sum")
};
assert!(sum.is_monotonic);
assert_eq!(
sum.data_points[0].value,
Some(number_data_point::Value::AsInt(7))
);
assert_eq!(sum.data_points[0].attributes[0].key, "method");
}

#[tokio::test]
async fn metrics_reuse_headers_and_mutual_tls_over_both_protocols() {
crate::test_environment::Environment::new()
.remove("OTEL_EXPORTER_OTLP_HEADERS")
.remove("OTEL_EXPORTER_OTLP_METRICS_HEADERS")
.run_async(async {
let pki = TestPki::generate();
let server_tls = ServerTls::mutual(&pki);
let tls = OtlpTls::from_pem(
Some(pki.ca_pem.as_bytes()),
Some((
pki.client.cert_pem.as_bytes(),
pki.client.key_pem.as_bytes(),
)),
)
.unwrap();
for protocol in [Protocol::Grpc, Protocol::HttpProtobuf] {
let grpc = OtlpTestServer::start_with_tls(&server_tls).await;
let mut http = OtlpHttpTestServer::start_with_tls(&server_tls).await;
let endpoint = if protocol == Protocol::Grpc {
grpc.endpoint()
} else {
http.endpoint()
};
let provider = target(endpoint, protocol)
.with_tls(tls.clone())
.with_headers(crate::parse_otlp_headers("authorization=Bearer%20metrics"))
.meter_provider()
.unwrap();
provider
.meter("test")
.u64_counter("requests_total")
.build()
.add(7, &[KeyValue::new("method", "GET")]);
let flush = {
let provider = provider.clone();
tokio::task::spawn_blocking(move || provider.force_flush())
};
flush.await.unwrap().unwrap();
tokio::task::spawn_blocking(move || provider.shutdown())
.await
.unwrap()
.unwrap();
if protocol == Protocol::Grpc {
let received = grpc.shutdown().await;
assert_eq!(received.metric_authorizations[0], "Bearer metrics");
assert_counter(&received.metric_exports[0]);
} else {
let export = http.next_export().await;
assert_eq!(export.path, "/v1/metrics");
assert_eq!(export.headers["authorization"], "Bearer metrics");
assert_counter(&ExportMetricsServiceRequest::decode(export.body).unwrap());
grpc.shutdown().await;
}
}
})
.await;
}

#[tokio::test]
async fn metrics_export_periodically_without_scraping_and_flush_on_shutdown() {
crate::test_environment::Environment::new()
.set("OTEL_METRIC_EXPORT_INTERVAL", "25")
.set("OTEL_EXPORTER_OTLP_METRICS_HEADERS", "x-metrics=from-env")
.set(
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
"http://ignored.invalid:4318",
)
.run_async(async {
let mut collector = OtlpHttpTestServer::start().await;
let provider = target(
&format!("{}/prefix", collector.endpoint()),
Protocol::HttpProtobuf,
)
.meter_provider()
.unwrap();
let counter = provider.meter("test").u64_counter("requests_total").build();
counter.add(7, &[KeyValue::new("method", "GET")]);
for _ in 0..2 {
let export = collector.next_export().await;
assert_eq!(export.path, "/prefix/v1/metrics");
assert_eq!(export.headers["x-metrics"], "from-env");
assert_counter(&ExportMetricsServiceRequest::decode(export.body).unwrap());
}
tokio::task::spawn_blocking(move || provider.shutdown())
.await
.unwrap()
.unwrap();
collector.next_export().await;
})
.await;
}
}
3 changes: 2 additions & 1 deletion crates/openshell-server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,15 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tracing-appender = { workspace = true }

# OpenTelemetry (OTLP trace and OCSF log export, opt-in via [openshell.gateway.otlp])
# OpenTelemetry (OTLP export, opt-in via [openshell.gateway.otlp])
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true }
tracing-opentelemetry = { workspace = true }

# Metrics
metrics = { workspace = true }
metrics-exporter-prometheus = { workspace = true }
metrics-util = { workspace = true, features = ["registry"] }

# Utilities
base64 = { workspace = true }
Expand Down
Loading
Loading