From bbcca1bcb37d904911a5e8a01319a7a36226bc7e Mon Sep 17 00:00:00 2001 From: DABH Date: Fri, 31 Jul 2026 15:11:14 -0500 Subject: [PATCH 01/24] Add replay-safe OpenTelemetry meter provider Google ADK and similar libraries record OpenTelemetry metrics through the process-global meter provider from code that runs workflow-side, so every workflow replay re-records them. ReplaySafeMeterProvider wraps a user-supplied MeterProvider and drops synchronous instrument recordings made from workflow code during replay, matching the first-execution-only semantics of workflow.metric_meter(). Observable instruments and non-workflow recordings pass through untouched. --- temporalio/contrib/opentelemetry/README.md | 31 +++ temporalio/contrib/opentelemetry/__init__.py | 8 +- .../contrib/opentelemetry/_meter_provider.py | 254 ++++++++++++++++++ .../opentelemetry/test_meter_provider.py | 100 +++++++ 4 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 temporalio/contrib/opentelemetry/_meter_provider.py create mode 100644 tests/contrib/opentelemetry/test_meter_provider.py diff --git a/temporalio/contrib/opentelemetry/README.md b/temporalio/contrib/opentelemetry/README.md index 2c6e39817..f436b7434 100644 --- a/temporalio/contrib/opentelemetry/README.md +++ b/temporalio/contrib/opentelemetry/README.md @@ -236,6 +236,37 @@ with tracer.start_as_current_span("my-operation") as span: }) ``` +## Replay-Safe Metrics + +For Temporal SDK metrics inside workflows, use `temporalio.workflow.metric_meter()`, +which is already replay-safe. However, third-party libraries (e.g. Google ADK) may +record OpenTelemetry metrics through the process-global meter provider from code +that runs inside workflows. Workflow code re-executes on every replay (cache +eviction, worker restart, redeploy), so a plain global meter provider re-records +those metrics on each replay, inflating counts. + +`ReplaySafeMeterProvider` wraps your meter provider so synchronous instrument +recordings made from workflow code are dropped during replay, mirroring what +`create_tracer_provider()` does for spans: + +```python +import opentelemetry.metrics +from opentelemetry.sdk.metrics import MeterProvider +from temporalio.contrib.opentelemetry import ReplaySafeMeterProvider + +# set_meter_provider only takes effect once per process, so this wrapper must +# be the first and only global meter provider set, installed before any +# library records metrics. +opentelemetry.metrics.set_meter_provider( + ReplaySafeMeterProvider(MeterProvider(metric_readers=[my_reader])) +) +``` + +Recordings are first-execution-only, matching `workflow.metric_meter()`: a +retried workflow task re-executes live and can record again. Observable +(asynchronous) instruments and recordings made outside workflows pass through +untouched. + ## Best Practices 1. **Register on Client**: Always register plugins/interceptors on the client, not the worker, to ensure proper context propagation diff --git a/temporalio/contrib/opentelemetry/__init__.py b/temporalio/contrib/opentelemetry/__init__.py index 74f069322..0407e1cda 100644 --- a/temporalio/contrib/opentelemetry/__init__.py +++ b/temporalio/contrib/opentelemetry/__init__.py @@ -9,14 +9,20 @@ TracingInterceptor, TracingWorkflowInboundInterceptor, ) +from temporalio.contrib.opentelemetry._meter_provider import ReplaySafeMeterProvider from temporalio.contrib.opentelemetry._otel_interceptor import OpenTelemetryInterceptor from temporalio.contrib.opentelemetry._plugin import OpenTelemetryPlugin -from temporalio.contrib.opentelemetry._tracer_provider import create_tracer_provider +from temporalio.contrib.opentelemetry._tracer_provider import ( + ReplaySafeTracerProvider, + create_tracer_provider, +) __all__ = [ "TracingInterceptor", "TracingWorkflowInboundInterceptor", "OpenTelemetryInterceptor", "OpenTelemetryPlugin", + "ReplaySafeMeterProvider", + "ReplaySafeTracerProvider", "create_tracer_provider", ] diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py new file mode 100644 index 000000000..0d1959243 --- /dev/null +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -0,0 +1,254 @@ +from collections.abc import Sequence +from typing import Any + +from opentelemetry.context import Context +from opentelemetry.metrics import ( + CallbackT, + Counter, + Histogram, + Meter, + MeterProvider, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, + _Gauge, +) +from opentelemetry.util.types import Attributes + +from temporalio import workflow + + +def _skip_recording() -> bool: + # in_workflow() must be evaluated first: is_replaying() requires an active + # workflow context. + return workflow.in_workflow() and workflow.unsafe.is_replaying() + + +class _ReplaySafeCounter(Counter): + def __init__(self, counter: Counter) -> None: + self._counter = counter + + def __getattr__(self, name: str) -> object: + return getattr(self._counter, name) + + def add( + self, + amount: int | float, + attributes: Attributes | None = None, + context: Context | None = None, + ) -> None: + if _skip_recording(): + # Skip recording metrics during workflow replay to avoid duplicate telemetry + return + self._counter.add(amount, attributes, context) + + +class _ReplaySafeUpDownCounter(UpDownCounter): + def __init__(self, counter: UpDownCounter) -> None: + self._counter = counter + + def __getattr__(self, name: str) -> object: + return getattr(self._counter, name) + + def add( + self, + amount: int | float, + attributes: Attributes | None = None, + context: Context | None = None, + ) -> None: + if _skip_recording(): + # Skip recording metrics during workflow replay to avoid duplicate telemetry + return + self._counter.add(amount, attributes, context) + + +class _ReplaySafeHistogram(Histogram): + def __init__(self, histogram: Histogram) -> None: + self._histogram = histogram + + def __getattr__(self, name: str) -> object: + return getattr(self._histogram, name) + + def record( + self, + amount: int | float, + attributes: Attributes | None = None, + context: Context | None = None, + ) -> None: + if _skip_recording(): + # Skip recording metrics during workflow replay to avoid duplicate telemetry + return + self._histogram.record(amount, attributes, context) + + +class _ReplaySafeGauge(_Gauge): + def __init__(self, gauge: _Gauge) -> None: + self._gauge = gauge + + def __getattr__(self, name: str) -> object: + return getattr(self._gauge, name) + + def set( + self, + amount: int | float, + attributes: Attributes | None = None, + context: Context | None = None, + ) -> None: + if _skip_recording(): + # Skip recording metrics during workflow replay to avoid duplicate telemetry + return + self._gauge.set(amount, attributes, context) + + +class _ReplaySafeMeter(Meter): + def __init__(self, meter: Meter) -> None: + super().__init__(meter.name, version=meter.version, schema_url=meter.schema_url) + self._meter = meter + + def create_counter( + self, + name: str, + unit: str = "", + description: str = "", + ) -> Counter: + return _ReplaySafeCounter(self._meter.create_counter(name, unit, description)) + + def create_up_down_counter( + self, + name: str, + unit: str = "", + description: str = "", + ) -> UpDownCounter: + return _ReplaySafeUpDownCounter( + self._meter.create_up_down_counter(name, unit, description) + ) + + def create_histogram( + self, + name: str, + unit: str = "", + description: str = "", + *, + explicit_bucket_boundaries_advisory: Sequence[float] | None = None, + ) -> Histogram: + return _ReplaySafeHistogram( + self._meter.create_histogram( + name, + unit, + description, + explicit_bucket_boundaries_advisory=explicit_bucket_boundaries_advisory, + ) + ) + + def create_gauge( + self, + name: str, + unit: str = "", + description: str = "", + ) -> _Gauge: + return _ReplaySafeGauge(self._meter.create_gauge(name, unit, description)) + + # Observable instruments pass through unwrapped: their callbacks run on the + # metric reader's collect thread, never inside workflow code. + + def create_observable_counter( + self, + name: str, + callbacks: Sequence[CallbackT] | None = None, + unit: str = "", + description: str = "", + ) -> ObservableCounter: + return self._meter.create_observable_counter(name, callbacks, unit, description) + + def create_observable_gauge( + self, + name: str, + callbacks: Sequence[CallbackT] | None = None, + unit: str = "", + description: str = "", + ) -> ObservableGauge: + return self._meter.create_observable_gauge(name, callbacks, unit, description) + + def create_observable_up_down_counter( + self, + name: str, + callbacks: Sequence[CallbackT] | None = None, + unit: str = "", + description: str = "", + ) -> ObservableUpDownCounter: + return self._meter.create_observable_up_down_counter( + name, callbacks, unit, description + ) + + +class ReplaySafeMeterProvider(MeterProvider): + """A meter provider that is safe for use during workflow replay. + + .. warning:: + This class is experimental and may change in future versions. + Use with caution in production environments. + + This meter provider wraps an OpenTelemetry MeterProvider and drops + synchronous instrument recordings (counter ``add()``, up-down counter + ``add()``, histogram ``record()``, and gauge ``set()``) made from workflow + code while the workflow is replaying. Without this, libraries that record + metrics from workflow code (e.g. ``google-adk``) re-record every + measurement on each replay, inflating counts. + + Recordings are therefore first-execution-only, matching + :py:meth:`temporalio.workflow.metric_meter`: a workflow task retry + re-executes live and can record again. Observable (asynchronous) + instruments pass through untouched since their callbacks run on the metric + reader's collect thread, never inside workflow code. Recordings outside + workflows are unaffected. + + Install this as the process-global meter provider before any library + (e.g. ``google-adk``) creates instruments:: + + opentelemetry.metrics.set_meter_provider( + ReplaySafeMeterProvider(my_meter_provider) + ) + + OpenTelemetry proxy meters late-bind, so calling ``set_meter_provider`` + after such libraries are imported still routes their instruments through + this wrapper. However, ``set_meter_provider`` only takes effect once per + process, so this wrapper must be the one and only global meter provider + ever set. + """ + + def __init__(self, meter_provider: MeterProvider) -> None: + """Initialize the replay-safe meter provider. + + Args: + meter_provider: The underlying OpenTelemetry MeterProvider to wrap. + """ + self._meter_provider = meter_provider + + def __getattr__(self, name: str) -> Any: + """Delegate all other attributes (e.g. ``shutdown``, ``force_flush``) + to the underlying meter provider. + """ + return getattr(self._meter_provider, name) + + def get_meter( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes | None = None, + ) -> Meter: + """Get a replay-safe meter from the underlying provider. + + Args: + name: The name of the instrumenting module. + version: The version string of the instrumenting library. + schema_url: The schema URL for the meter. + attributes: Additional attributes for the meter. + + Returns: + A replay-safe meter instance. + """ + return _ReplaySafeMeter( + self._meter_provider.get_meter(name, version, schema_url, attributes) + ) diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py new file mode 100644 index 000000000..44f1b8760 --- /dev/null +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -0,0 +1,100 @@ +"""Unit tests for ReplaySafeMeterProvider outside workflows.""" + +from collections.abc import Iterable + +from opentelemetry.metrics import CallbackOptions, Meter, MeterProvider, Observation +from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.util.types import Attributes + +from temporalio.contrib.opentelemetry import ReplaySafeMeterProvider + + +def _metric_data_points(reader: InMemoryMetricReader) -> dict[str, list]: + points: dict[str, list] = {} + data = reader.get_metrics_data() + assert data is not None + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + for metric in sm.metrics: + points.setdefault(metric.name, []).extend(metric.data.data_points) + return points + + +def test_replay_safe_meter_provider_sync_instruments_pass_through_outside_workflow(): + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + meter = provider.get_meter("test-meter") + + meter.create_counter("counter").add(2, {"attr": "val"}) + meter.create_up_down_counter("up_down_counter").add(-3) + meter.create_histogram("histogram").record(4) + meter.create_gauge("gauge").set(5) + + points = _metric_data_points(reader) + assert points["counter"][0].value == 2 + assert dict(points["counter"][0].attributes) == {"attr": "val"} + assert points["up_down_counter"][0].value == -3 + assert points["histogram"][0].count == 1 + assert points["histogram"][0].sum == 4 + assert points["gauge"][0].value == 5 + + +def test_replay_safe_meter_provider_observable_instruments_pass_through(): + def callback(options: CallbackOptions) -> Iterable[Observation]: # type: ignore[reportUnusedParameter] + return [Observation(10)] + + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + meter = provider.get_meter("test-meter") + + meter.create_observable_counter("observable_counter", callbacks=[callback]) + meter.create_observable_gauge("observable_gauge", callbacks=[callback]) + meter.create_observable_up_down_counter( + "observable_up_down_counter", callbacks=[callback] + ) + + points = _metric_data_points(reader) + assert points["observable_counter"][0].value == 10 + assert points["observable_gauge"][0].value == 10 + assert points["observable_up_down_counter"][0].value == 10 + + +def test_replay_safe_meter_provider_delegates_get_meter_arguments(): + class RecordingMeterProvider(MeterProvider): + def __init__(self) -> None: + self.calls: list[tuple] = [] + self._inner = SdkMeterProvider() + + def get_meter( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes | None = None, + ) -> Meter: + self.calls.append((name, version, schema_url, attributes)) + return self._inner.get_meter(name, version, schema_url, attributes) + + inner_provider = RecordingMeterProvider() + provider = ReplaySafeMeterProvider(inner_provider) + meter = provider.get_meter( + "test-meter", + version="1.2.3", + schema_url="https://example.com/schema", + attributes={"attr": "val"}, + ) + + assert inner_provider.calls == [ + ("test-meter", "1.2.3", "https://example.com/schema", {"attr": "val"}) + ] + assert meter.name == "test-meter" + assert meter.version == "1.2.3" + assert meter.schema_url == "https://example.com/schema" + + +def test_replay_safe_meter_provider_delegates_other_attributes(): + inner_provider = SdkMeterProvider() + provider = ReplaySafeMeterProvider(inner_provider) + assert provider.force_flush() + provider.shutdown() From 6d588de3e472086c0fb2ebb566272a7861a5f61c Mon Sep 17 00:00:00 2001 From: DABH Date: Fri, 31 Jul 2026 15:11:28 -0500 Subject: [PATCH 02/24] Warn on replay-unsafe global OTel providers in GoogleAdkPlugin At worker configuration, warn when the global OpenTelemetry meter or tracer provider is not replay-safe, pointing users at ReplaySafeMeterProvider and create_tracer_provider. Add a regression test proving 1 real execution + 3 replays leaves ADK metric instruments at their nonzero baseline with ReplaySafeMeterProvider installed, plus a control asserting the 4x inflation without it. --- CHANGELOG.md | 10 + .../contrib/google_adk_agents/README.md | 49 +++ .../contrib/google_adk_agents/_plugin.py | 51 ++++ tests/conftest.py | 11 + .../google_adk_agents/test_replay_metrics.py | 289 ++++++++++++++++++ 5 files changed, 410 insertions(+) create mode 100644 tests/contrib/google_adk_agents/test_replay_metrics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 483c73d1d..7854ccbfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,16 @@ to include examples, links to docs, or any other relevant information. ### Added +- Added experimental `temporalio.contrib.opentelemetry.ReplaySafeMeterProvider`, a wrapping + OpenTelemetry `MeterProvider` that drops synchronous instrument recordings made from workflow + code during replay. Install it as the process-global meter provider when libraries record + OpenTelemetry metrics from workflow code (e.g. Google ADK) so that workflow replays (cache + eviction, worker restarts, redeploys) do not re-record metrics; recordings are + first-execution-only, matching `temporalio.workflow.metric_meter()`. + `temporalio.contrib.opentelemetry.ReplaySafeTracerProvider` is now also exported. + `GoogleAdkPlugin` now warns at worker configuration time when the global OpenTelemetry meter + or tracer provider is not replay-safe. + ### Changed ### Deprecated diff --git a/temporalio/contrib/google_adk_agents/README.md b/temporalio/contrib/google_adk_agents/README.md index 92fd1cea7..54c55938a 100644 --- a/temporalio/contrib/google_adk_agents/README.md +++ b/temporalio/contrib/google_adk_agents/README.md @@ -199,6 +199,55 @@ agent = Agent( ) ``` +## Telemetry and Workflow Replay + +ADK records OpenTelemetry metrics (scope `gcp.vertex.agent`, e.g. +`gen_ai.client.token.usage`) and spans through the process-global +OpenTelemetry providers from code that runs inside the workflow. Workflow +code re-executes on every replay, so with a plain global provider each replay +re-records all of that telemetry even though no model or tool actually ran +again — for example, 1 real execution followed by 3 replays yields 4x the +observations on every instrument. Replays happen routinely in production: +workflow cache eviction, worker restarts, redeploys, or running with +`max_cached_workflows=0`. + +To avoid this, install Temporal's replay-safe providers as the global +OpenTelemetry providers. They pass recordings through on first execution and +drop them during replay: + +```python +import opentelemetry.metrics +import opentelemetry.trace +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from temporalio.contrib.opentelemetry import ( + ReplaySafeMeterProvider, + create_tracer_provider, +) + +# set_meter_provider/set_tracer_provider only take effect once per process, +# so these wrappers must be the first and only global providers set. +opentelemetry.metrics.set_meter_provider( + ReplaySafeMeterProvider( + MeterProvider(metric_readers=[PeriodicExportingMetricReader(my_exporter)]) + ) +) +tracer_provider = create_tracer_provider() +tracer_provider.add_span_processor(BatchSpanProcessor(my_span_exporter)) +opentelemetry.trace.set_tracer_provider(tracer_provider) +``` + +`GoogleAdkPlugin` warns at worker configuration time when a global provider +is installed that is not replay-safe. + +Recordings are first-execution-only, matching +`temporalio.workflow.metric_meter()`: a retried workflow task re-executes +live and can record again, and tokens consumed by failed activity attempts +are not counted. Telemetry recorded from activities (worker-side) is +unaffected. + ## Integration Points This integration provides comprehensive support for running Google ADK Agents within Temporal workflows while maintaining: diff --git a/temporalio/contrib/google_adk_agents/_plugin.py b/temporalio/contrib/google_adk_agents/_plugin.py index 7344485c8..1d95f1fce 100644 --- a/temporalio/contrib/google_adk_agents/_plugin.py +++ b/temporalio/contrib/google_adk_agents/_plugin.py @@ -3,16 +3,27 @@ import dataclasses import time import uuid +import warnings from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager from typing import Any +import opentelemetry.metrics +import opentelemetry.trace +from opentelemetry.metrics import NoOpMeterProvider +from opentelemetry.metrics._internal import _ProxyMeterProvider +from opentelemetry.trace import NoOpTracerProvider, ProxyTracerProvider + from temporalio import workflow from temporalio.contrib.google_adk_agents._mcp import TemporalMcpToolSetProvider from temporalio.contrib.google_adk_agents._model import ( invoke_model, invoke_model_streaming, ) +from temporalio.contrib.opentelemetry import ( + ReplaySafeMeterProvider, + ReplaySafeTracerProvider, +) from temporalio.contrib.pydantic import ( PydanticPayloadConverter, ToJsonOptions, @@ -20,11 +31,44 @@ from temporalio.converter import DataConverter, DefaultPayloadConverter from temporalio.plugin import SimplePlugin from temporalio.worker import ( + WorkerConfig, WorkflowRunner, ) from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner +def _warn_if_global_otel_providers_not_replay_safe() -> None: + # ADK records metrics and spans through the process-global OpenTelemetry + # providers from code that runs workflow-side, so a non-replay-safe global + # provider re-emits that telemetry on every workflow replay. Unset (proxy) + # and no-op providers drop recordings and are fine. + meter_provider = opentelemetry.metrics.get_meter_provider() + if not isinstance( + meter_provider, + (ReplaySafeMeterProvider, NoOpMeterProvider, _ProxyMeterProvider), + ): + warnings.warn( + "The global OpenTelemetry MeterProvider is not replay-safe: Google ADK " + "records metrics from workflow code, so every workflow replay will " + "re-record them. Wrap your provider in " + "temporalio.contrib.opentelemetry.ReplaySafeMeterProvider and make it " + "the first and only global provider set: " + "opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(provider))" + ) + tracer_provider = opentelemetry.trace.get_tracer_provider() + if not isinstance( + tracer_provider, + (ReplaySafeTracerProvider, NoOpTracerProvider, ProxyTracerProvider), + ): + warnings.warn( + "The global OpenTelemetry TracerProvider is not replay-safe: Google ADK " + "creates spans from workflow code, so every workflow replay will " + "re-emit them. Install a replay-safe provider: " + "opentelemetry.trace.set_tracer_provider(" + "temporalio.contrib.opentelemetry.create_tracer_provider())" + ) + + def setup_deterministic_runtime(): """Configures ADK runtime for Temporal determinism. @@ -118,6 +162,13 @@ def workflow_runner(runner: WorkflowRunner | None) -> WorkflowRunner: workflow_runner=workflow_runner, ) + def configure_worker(self, config: WorkerConfig) -> WorkerConfig: + """See base class. Also warns when the global OpenTelemetry providers + are not replay-safe, since ADK telemetry would duplicate on replay. + """ + _warn_if_global_otel_providers_not_replay_safe() + return super().configure_worker(config) + def _configure_data_converter( self, converter: DataConverter | None ) -> DataConverter: diff --git a/tests/conftest.py b/tests/conftest.py index 9c57bc0d1..f4d01632a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import sys from collections.abc import AsyncGenerator, Iterator +import opentelemetry.metrics._internal import opentelemetry.trace import pytest import pytest_asyncio @@ -268,3 +269,13 @@ def reset_otel_tracer_provider(): yield opentelemetry.trace._TRACER_PROVIDER_SET_ONCE = Once() opentelemetry.trace._TRACER_PROVIDER = None + + +@pytest.fixture +def reset_otel_meter_provider(): + """Reset global OpenTelemetry meter provider state around tests.""" + opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() + opentelemetry.metrics._internal._METER_PROVIDER = None + yield + opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() + opentelemetry.metrics._internal._METER_PROVIDER = None diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py new file mode 100644 index 000000000..e6e7189e3 --- /dev/null +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -0,0 +1,289 @@ +"""Tests for replay-safe handling of ADK's OpenTelemetry metrics. + +Google ADK records token-usage/latency metrics through the process-global +OpenTelemetry meter (scope "gcp.vertex.agent") from code that runs +workflow-side under the Temporal adapter. Without gating, 1 real execution + +N replays produces (1 + N) observations per instrument, while the activity +(the model / tool call) runs exactly once. Installing +ReplaySafeMeterProvider as the global meter provider suppresses the replay +recordings while leaving first-execution recordings intact. +""" + +import uuid +import warnings +from collections.abc import AsyncGenerator +from datetime import timedelta + +import opentelemetry.metrics +import pytest +from google.adk import Agent +from google.adk.models import BaseLlm, LLMRegistry +from google.adk.models.llm_request import LlmRequest +from google.adk.models.llm_response import LlmResponse +from google.adk.runners import InMemoryRunner +from google.adk.utils.context_utils import Aclosing +from google.genai import types +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import set_tracer_provider + +import temporalio.contrib.google_adk_agents.workflow +from temporalio import activity, workflow +from temporalio.client import Client, WorkflowHistory +from temporalio.contrib.google_adk_agents import GoogleAdkPlugin, TemporalModel +from temporalio.contrib.opentelemetry import ( + ReplaySafeMeterProvider, + create_tracer_provider, +) +from temporalio.worker import Replayer, Worker, WorkerConfig +from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner + +MODEL_NAME = "replay-metrics-model" +ADK_METER_SCOPE = "gcp.vertex.agent" + +# One agent invocation, one tool call, two model calls (tool call + final +# answer), token usage recorded twice per model call (input + output). +EXPECTED_BASELINE = { + "gen_ai.invoke_agent.duration": 1, + "gen_ai.invoke_agent.inference_calls": 1, + "gen_ai.invoke_agent.tool_calls": 1, + "gen_ai.execute_tool.duration": 1, + "gen_ai.client.operation.duration": 2, + "gen_ai.client.token.usage": 4, +} + +# Counts real (worker-side) activity executions; replays must not add to it. +activity_executions = 0 + + +@activity.defn +async def replay_metrics_get_weather(city: str) -> str: # type: ignore[reportUnusedParameter] + global activity_executions + activity_executions += 1 + return "Warm and sunny. 17 degrees." + + +class ReplayMetricsModel(BaseLlm): + """Scripted model: one tool call, then a final text answer. + + Both responses carry usage_metadata so gen_ai.client.token.usage records. + """ + + @classmethod + def supported_models(cls) -> list[str]: + return [MODEL_NAME] + + def _responses(self) -> list[LlmResponse]: + return [ + LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + args={"city": "New York"}, + name="replay_metrics_get_weather", + ) + ) + ], + ), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=100, + candidates_token_count=25, + total_token_count=125, + ), + ), + LlmResponse( + content=types.Content( + role="model", + parts=[types.Part(text="warm and sunny")], + ), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=200, + candidates_token_count=10, + total_token_count=210, + ), + ), + ] + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + for response in self._responses(): + if any(content == response.content for content in llm_request.contents): + continue + yield response + return + + +@workflow.defn +class ReplayMetricsAgent: + @workflow.run + async def run(self, prompt: str, model_name: str) -> str | None: + weather_tool = temporalio.contrib.google_adk_agents.workflow.activity_tool( + replay_metrics_get_weather, + start_to_close_timeout=timedelta(seconds=60), + ) + agent = Agent( + name="replay_metrics_agent", + model=TemporalModel(model_name), + tools=[weather_tool], + ) + runner = InMemoryRunner(agent=agent, app_name="replay_metrics_app") + session = await runner.session_service.create_session( + app_name="replay_metrics_app", user_id="test" + ) + last_text = None + async with Aclosing( + runner.run_async( + user_id="test", + session_id=session.id, + new_message=types.Content(role="user", parts=[types.Part(text=prompt)]), + ) + ) as agen: + async for event in agen: + if ( + event.content + and event.content.parts + and event.content.parts[0].text + ): + last_text = event.content.parts[0].text + return last_text + + +def adk_metric_counts(reader: InMemoryMetricReader) -> dict[str, int]: + """Total observation count per ADK instrument (sum of data-point counts).""" + counts: dict[str, int] = {} + data = reader.get_metrics_data() + if data is None: + return counts + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + if sm.scope.name != ADK_METER_SCOPE: + continue + for metric in sm.metrics: + for dp in getattr(metric.data, "data_points", []): + counts[metric.name] = counts.get(metric.name, 0) + getattr( + dp, "count", 1 + ) + return counts + + +async def _run_once_and_replay( + client: Client, num_replays: int +) -> tuple[int, WorkflowHistory]: + """Run the agent workflow once for real, then replay it num_replays times. + + Returns the number of real activity executions observed for this run. + """ + LLMRegistry.register(ReplayMetricsModel) + + new_config = client.config() + new_config["plugins"] = [GoogleAdkPlugin()] + client = Client(**new_config) + + activity_executions_before = activity_executions + task_queue = f"replay-metrics-{uuid.uuid4()}" + # Deliberately not setting max_cached_workflows=0 so the live run is + # exactly one real execution with no forced replay per workflow task. + async with Worker( + client, + task_queue=task_queue, + activities=[replay_metrics_get_weather], + workflows=[ReplayMetricsAgent], + ): + handle = await client.start_workflow( + ReplayMetricsAgent.run, + args=["What is the weather in New York?", MODEL_NAME], + id=f"replay-metrics-{uuid.uuid4()}", + task_queue=task_queue, + execution_timeout=timedelta(seconds=60), + ) + result = await handle.result() + assert result == "warm and sunny" + history = await handle.fetch_history() + + for _ in range(num_replays): + await Replayer( + workflows=[ReplayMetricsAgent], + plugins=[GoogleAdkPlugin()], + ).replay_workflow(history) + + return activity_executions - activity_executions_before, history + + +async def test_replay_safe_meter_provider_suppresses_replay_metrics( + client: Client, + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] +): + reader = InMemoryMetricReader() + opentelemetry.metrics.set_meter_provider( + ReplaySafeMeterProvider(MeterProvider(metric_readers=[reader])) + ) + + real_executions, _ = await _run_once_and_replay(client, num_replays=3) + + # First execution recorded exactly once (not suppressed), replays added + # zero observations, and the activity never re-executed. + assert real_executions == 1 + assert adk_metric_counts(reader) == EXPECTED_BASELINE + + +async def test_replay_metrics_duplicate_without_replay_safe_meter_provider( + client: Client, + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] +): + # Control: without the wrapper, every replay re-records every + # workflow-side ADK metric even though nothing really re-executed. + reader = InMemoryMetricReader() + opentelemetry.metrics.set_meter_provider(MeterProvider(metric_readers=[reader])) + + real_executions, _ = await _run_once_and_replay(client, num_replays=3) + + assert real_executions == 1 + assert adk_metric_counts(reader) == { + name: count * (1 + 3) for name, count in EXPECTED_BASELINE.items() + } + + +def _worker_config() -> WorkerConfig: + return WorkerConfig(workflow_runner=SandboxedWorkflowRunner()) + + +def test_plugin_warns_on_non_replay_safe_meter_provider( + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] +): + opentelemetry.metrics.set_meter_provider(MeterProvider()) + with pytest.warns(UserWarning, match="MeterProvider is not replay-safe"): + GoogleAdkPlugin().configure_worker(_worker_config()) + + +def test_plugin_warns_on_non_replay_safe_tracer_provider( + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + set_tracer_provider(TracerProvider()) + with pytest.warns(UserWarning, match="TracerProvider is not replay-safe"): + GoogleAdkPlugin().configure_worker(_worker_config()) + + +def test_plugin_does_not_warn_with_replay_safe_providers( + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(MeterProvider())) + set_tracer_provider(create_tracer_provider()) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + GoogleAdkPlugin().configure_worker(_worker_config()) + assert not [w for w in recorded if "replay-safe" in str(w.message)] + + +def test_plugin_does_not_warn_with_unset_providers( + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + GoogleAdkPlugin().configure_worker(_worker_config()) + assert not [w for w in recorded if "replay-safe" in str(w.message)] From 0202618d4c6dd2ad18248878c24e0c36453052f0 Mon Sep 17 00:00:00 2001 From: DABH Date: Fri, 31 Jul 2026 15:31:35 -0500 Subject: [PATCH 03/24] Forward newer OTel meter parameters only when set ReplaySafeMeterProvider unconditionally forwarded get_meter attributes (added in opentelemetry 1.26) and create_histogram explicit_bucket_boundaries_advisory (added in 1.30), raising TypeError into caller code on older APIs within the supported range. Forward them only when non-None, matching unwrapped-caller behavior. --- .../contrib/opentelemetry/_meter_provider.py | 32 ++++++++---- tests/conftest.py | 8 ++- .../opentelemetry/test_meter_provider.py | 52 ++++++++++++++++++- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index 0d1959243..bb93e14d1 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -102,6 +102,10 @@ def set( class _ReplaySafeMeter(Meter): + # Overrides every Meter method as of opentelemetry-api 1.42. OTel adds new + # instrument kinds as non-abstract no-op defaults on the Meter ABC (e.g. + # create_gauge in 1.23), which __getattr__ cannot intercept, so new Meter + # methods must be audited and overridden here on opentelemetry upgrades. def __init__(self, meter: Meter) -> None: super().__init__(meter.name, version=meter.version, schema_url=meter.schema_url) self._meter = meter @@ -132,13 +136,16 @@ def create_histogram( *, explicit_bucket_boundaries_advisory: Sequence[float] | None = None, ) -> Histogram: - return _ReplaySafeHistogram( - self._meter.create_histogram( - name, - unit, - description, - explicit_bucket_boundaries_advisory=explicit_bucket_boundaries_advisory, + # Forward the advisory only when set: the parameter was added in + # opentelemetry 1.30 and unconditionally forwarding it raises TypeError + # on older APIs still within the supported version range. + kwargs: dict[str, Any] = {} + if explicit_bucket_boundaries_advisory is not None: + kwargs["explicit_bucket_boundaries_advisory"] = ( + explicit_bucket_boundaries_advisory ) + return _ReplaySafeHistogram( + self._meter.create_histogram(name, unit, description, **kwargs) ) def create_gauge( @@ -249,6 +256,13 @@ def get_meter( Returns: A replay-safe meter instance. """ - return _ReplaySafeMeter( - self._meter_provider.get_meter(name, version, schema_url, attributes) - ) + # Forward attributes only when set: the parameter was added in + # opentelemetry 1.26 and passing it to older providers raises + # TypeError. + if attributes is None: + inner = self._meter_provider.get_meter(name, version, schema_url) + else: + inner = self._meter_provider.get_meter( + name, version, schema_url, attributes + ) + return _ReplaySafeMeter(inner) diff --git a/tests/conftest.py b/tests/conftest.py index f4d01632a..1bc3c37ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -273,7 +273,13 @@ def reset_otel_tracer_provider(): @pytest.fixture def reset_otel_meter_provider(): - """Reset global OpenTelemetry meter provider state around tests.""" + """Reset global OpenTelemetry meter provider state around tests. + + Proxy meters/instruments already bound to a real provider stay bound after + this reset; only the next set_meter_provider call rebinds them. Tests must + not assume an unset global provider drops recordings from instruments + created in earlier tests. + """ opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() opentelemetry.metrics._internal._METER_PROVIDER = None yield diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py index 44f1b8760..23af2a023 100644 --- a/tests/contrib/opentelemetry/test_meter_provider.py +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -2,7 +2,15 @@ from collections.abc import Iterable -from opentelemetry.metrics import CallbackOptions, Meter, MeterProvider, Observation +from opentelemetry.metrics import ( + CallbackOptions, + Histogram, + Meter, + MeterProvider, + NoOpMeter, + NoOpMeterProvider, + Observation, +) from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.util.types import Attributes @@ -93,6 +101,48 @@ def get_meter( assert meter.schema_url == "https://example.com/schema" +def test_replay_safe_meter_provider_supports_older_otel_signatures(): + """Newer opentelemetry-api parameters (get_meter attributes, 1.26; + create_histogram explicit_bucket_boundaries_advisory, 1.30) must only be + forwarded when set, so providers with older signatures keep working.""" + + class Pre130Meter(NoOpMeter): + def __init__(self) -> None: + super().__init__("pre-1.30-meter") + self.histogram_calls: list[tuple[str, str, str]] = [] + + def create_histogram( # type: ignore[override] + self, + name: str, + unit: str = "", + description: str = "", + ) -> Histogram: + self.histogram_calls.append((name, unit, description)) + return super().create_histogram(name, unit, description) + + class Pre126MeterProvider(NoOpMeterProvider): + def __init__(self) -> None: + self.meter = Pre130Meter() + self.get_meter_calls: list[tuple[str, str | None, str | None]] = [] + + def get_meter( # type: ignore[override] + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + ) -> Meter: + self.get_meter_calls.append((name, version, schema_url)) + return self.meter + + inner_provider = Pre126MeterProvider() + provider = ReplaySafeMeterProvider(inner_provider) + meter = provider.get_meter("test-meter") + meter.create_histogram("histogram").record(1) + + assert inner_provider.get_meter_calls == [("test-meter", None, None)] + assert inner_provider.meter.histogram_calls == [("histogram", "", "")] + + def test_replay_safe_meter_provider_delegates_other_attributes(): inner_provider = SdkMeterProvider() provider = ReplaySafeMeterProvider(inner_provider) From 5a962699efcb784daa26947037ce67e7f4c71fd4 Mon Sep 17 00:00:00 2001 From: DABH Date: Fri, 31 Jul 2026 17:02:46 -0500 Subject: [PATCH 04/24] Harden replay-safe metrics against old opentelemetry-api - Guard the private _Gauge import (added in opentelemetry-api 1.23) and import ReplaySafeMeterProvider into the package lazily so temporalio.contrib.opentelemetry stays importable for tracing-only users on opentelemetry-api < 1.12, with an actionable error on access. - Classify the global meter provider without a module-level private import; when the private proxy class cannot be imported the provider is unclassifiable and no warning is issued. - Forward the context argument to sync instrument wrappers only when set; the parameter was added in opentelemetry-api 1.28 and older instruments reject it. - Add explicit UserWarning category and stacklevel so provider warnings point at the user's Worker(...) call. --- .../contrib/google_adk_agents/_plugin.py | 41 ++++- temporalio/contrib/opentelemetry/__init__.py | 28 +++- .../contrib/opentelemetry/_meter_provider.py | 33 +++- .../google_adk_agents/test_replay_metrics.py | 63 ++++++++ .../opentelemetry/test_meter_provider.py | 151 +++++++++++++++++- 5 files changed, 298 insertions(+), 18 deletions(-) diff --git a/temporalio/contrib/google_adk_agents/_plugin.py b/temporalio/contrib/google_adk_agents/_plugin.py index 1d95f1fce..44f8d073e 100644 --- a/temporalio/contrib/google_adk_agents/_plugin.py +++ b/temporalio/contrib/google_adk_agents/_plugin.py @@ -10,8 +10,7 @@ import opentelemetry.metrics import opentelemetry.trace -from opentelemetry.metrics import NoOpMeterProvider -from opentelemetry.metrics._internal import _ProxyMeterProvider +from opentelemetry.metrics import MeterProvider, NoOpMeterProvider from opentelemetry.trace import NoOpTracerProvider, ProxyTracerProvider from temporalio import workflow @@ -37,23 +36,47 @@ from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner +def _meter_provider_replay_safe(provider: MeterProvider) -> bool | None: + """Classify the global meter provider for replay safety. + + Returns True when replay-safe (including providers that drop all + recordings), False when positively identified as replay-unsafe, and None + when it cannot be classified, in which case no warning should be issued + since a false positive is worse than a missed warning. + """ + if isinstance(provider, (ReplaySafeMeterProvider, NoOpMeterProvider)): + return True + try: + # Unlike tracing's public ProxyTracerProvider, the proxy (unset) meter + # provider has no public counterpart. Import the private class lazily + # so a moved or removed symbol cannot break module import; it is + # present in opentelemetry-api 1.12 through at least 1.42. + from opentelemetry.metrics._internal import _ProxyMeterProvider + except ImportError: + return None + return isinstance(provider, _ProxyMeterProvider) + + def _warn_if_global_otel_providers_not_replay_safe() -> None: # ADK records metrics and spans through the process-global OpenTelemetry # providers from code that runs workflow-side, so a non-replay-safe global # provider re-emits that telemetry on every workflow replay. Unset (proxy) # and no-op providers drop recordings and are fine. + # + # stacklevel=4 attributes the warnings to the user's Worker(...) call: + # warn -> this helper -> GoogleAdkPlugin.configure_worker -> + # Worker.__init__ -> user code. meter_provider = opentelemetry.metrics.get_meter_provider() - if not isinstance( - meter_provider, - (ReplaySafeMeterProvider, NoOpMeterProvider, _ProxyMeterProvider), - ): + if _meter_provider_replay_safe(meter_provider) is False: warnings.warn( "The global OpenTelemetry MeterProvider is not replay-safe: Google ADK " "records metrics from workflow code, so every workflow replay will " "re-record them. Wrap your provider in " "temporalio.contrib.opentelemetry.ReplaySafeMeterProvider and make it " "the first and only global provider set: " - "opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(provider))" + "opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(provider))", + UserWarning, + stacklevel=4, ) tracer_provider = opentelemetry.trace.get_tracer_provider() if not isinstance( @@ -65,7 +88,9 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: "creates spans from workflow code, so every workflow replay will " "re-emit them. Install a replay-safe provider: " "opentelemetry.trace.set_tracer_provider(" - "temporalio.contrib.opentelemetry.create_tracer_provider())" + "temporalio.contrib.opentelemetry.create_tracer_provider())", + UserWarning, + stacklevel=4, ) diff --git a/temporalio/contrib/opentelemetry/__init__.py b/temporalio/contrib/opentelemetry/__init__.py index 0407e1cda..25dbb9a62 100644 --- a/temporalio/contrib/opentelemetry/__init__.py +++ b/temporalio/contrib/opentelemetry/__init__.py @@ -5,11 +5,24 @@ propagation for distributed tracing. """ +from typing import Any + from temporalio.contrib.opentelemetry._interceptor import ( TracingInterceptor, TracingWorkflowInboundInterceptor, ) -from temporalio.contrib.opentelemetry._meter_provider import ReplaySafeMeterProvider + +_meter_provider_import_error: ImportError | None = None +try: + from temporalio.contrib.opentelemetry._meter_provider import ( + ReplaySafeMeterProvider, + ) +except ImportError as err: + # opentelemetry-api < 1.12 has no opentelemetry.metrics module. Keep the + # tracing integration importable and raise a clear error only when + # ReplaySafeMeterProvider is actually accessed (see __getattr__ below). + _meter_provider_import_error = err + from temporalio.contrib.opentelemetry._otel_interceptor import OpenTelemetryInterceptor from temporalio.contrib.opentelemetry._plugin import OpenTelemetryPlugin from temporalio.contrib.opentelemetry._tracer_provider import ( @@ -26,3 +39,16 @@ "ReplaySafeTracerProvider", "create_tracer_provider", ] + + +def __getattr__(name: str) -> Any: + # Only reachable for ReplaySafeMeterProvider when the guarded import above + # failed; otherwise the module attribute exists and this is never called. + if name == "ReplaySafeMeterProvider": + raise ImportError( + "ReplaySafeMeterProvider requires the OpenTelemetry metrics API " + "(opentelemetry.metrics), which the installed opentelemetry-api " + "version does not provide. Install opentelemetry-api >= 1.12 " + "(>= 1.23 for synchronous gauge support)." + ) from _meter_provider_import_error + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index bb93e14d1..944f0f56c 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING, Any from opentelemetry.context import Context from opentelemetry.metrics import ( @@ -12,13 +12,32 @@ ObservableGauge, ObservableUpDownCounter, UpDownCounter, - _Gauge, ) from opentelemetry.util.types import Attributes from temporalio import workflow +def _forward_context_kwarg(context: Context | None) -> dict[str, Any]: + # Forward the context only when set: the parameter was added to the + # synchronous instrument methods in opentelemetry 1.28 and older + # instruments raise TypeError when it is passed. + return {} if context is None else {"context": context} + + +if TYPE_CHECKING: + from opentelemetry.metrics import _Gauge +else: + try: + from opentelemetry.metrics import _Gauge + except ImportError: + # Synchronous gauges (and Meter.create_gauge) were added in + # opentelemetry-api 1.23. Fall back to a plain base class so this + # module stays importable on older versions in the supported range; + # meters there have no create_gauge to wrap anyway. + _Gauge = object + + def _skip_recording() -> bool: # in_workflow() must be evaluated first: is_replaying() requires an active # workflow context. @@ -41,7 +60,7 @@ def add( if _skip_recording(): # Skip recording metrics during workflow replay to avoid duplicate telemetry return - self._counter.add(amount, attributes, context) + self._counter.add(amount, attributes, **_forward_context_kwarg(context)) class _ReplaySafeUpDownCounter(UpDownCounter): @@ -60,7 +79,7 @@ def add( if _skip_recording(): # Skip recording metrics during workflow replay to avoid duplicate telemetry return - self._counter.add(amount, attributes, context) + self._counter.add(amount, attributes, **_forward_context_kwarg(context)) class _ReplaySafeHistogram(Histogram): @@ -79,7 +98,7 @@ def record( if _skip_recording(): # Skip recording metrics during workflow replay to avoid duplicate telemetry return - self._histogram.record(amount, attributes, context) + self._histogram.record(amount, attributes, **_forward_context_kwarg(context)) class _ReplaySafeGauge(_Gauge): @@ -98,7 +117,7 @@ def set( if _skip_recording(): # Skip recording metrics during workflow replay to avoid duplicate telemetry return - self._gauge.set(amount, attributes, context) + self._gauge.set(amount, attributes, **_forward_context_kwarg(context)) class _ReplaySafeMeter(Meter): @@ -154,6 +173,8 @@ def create_gauge( unit: str = "", description: str = "", ) -> _Gauge: + # On opentelemetry-api < 1.23 the underlying meter has no create_gauge + # and this raises AttributeError, same as calling it on the raw meter. return _ReplaySafeGauge(self._meter.create_gauge(name, unit, description)) # Observable instruments pass through unwrapped: their callbacks run on the diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index e6e7189e3..5dba5dfb3 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -9,6 +9,7 @@ recordings while leaving first-execution recordings intact. """ +import sys import uuid import warnings from collections.abc import AsyncGenerator @@ -23,6 +24,8 @@ from google.adk.runners import InMemoryRunner from google.adk.utils.context_utils import Aclosing from google.genai import types +from opentelemetry.metrics import NoOpMeterProvider +from opentelemetry.metrics._internal import _ProxyMeterProvider from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.sdk.trace import TracerProvider @@ -287,3 +290,63 @@ def test_plugin_does_not_warn_with_unset_providers( warnings.simplefilter("always") GoogleAdkPlugin().configure_worker(_worker_config()) assert not [w for w in recorded if "replay-safe" in str(w.message)] + + +def test_meter_provider_replay_safety_classification(monkeypatch: pytest.MonkeyPatch): + from temporalio.contrib.google_adk_agents._plugin import ( + _meter_provider_replay_safe, + ) + + assert _meter_provider_replay_safe(ReplaySafeMeterProvider(MeterProvider())) is True + assert _meter_provider_replay_safe(NoOpMeterProvider()) is True + assert _meter_provider_replay_safe(_ProxyMeterProvider()) is True + assert _meter_provider_replay_safe(MeterProvider()) is False + + # When the private proxy class cannot be imported, providers other than + # the replay-safe/no-op ones are unclassifiable and must not warn. + monkeypatch.setitem(sys.modules, "opentelemetry.metrics._internal", None) + assert _meter_provider_replay_safe(MeterProvider()) is None + assert _meter_provider_replay_safe(NoOpMeterProvider()) is True + + +def test_plugin_does_not_warn_when_proxy_detection_unavailable( + monkeypatch: pytest.MonkeyPatch, + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + # A replay-unsafe provider is set, but with the private proxy class + # unimportable it cannot be classified, so the plugin stays silent rather + # than risking a false positive. + opentelemetry.metrics.set_meter_provider(MeterProvider()) + monkeypatch.setitem(sys.modules, "opentelemetry.metrics._internal", None) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + GoogleAdkPlugin().configure_worker(_worker_config()) + assert not [ + w for w in recorded if "MeterProvider is not replay-safe" in str(w.message) + ] + + +async def test_plugin_warning_points_at_worker_construction( + client: Client, + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + # stacklevel on the warning must attribute it to the user's Worker(...) + # call, i.e. this file, not SDK internals. + opentelemetry.metrics.set_meter_provider(MeterProvider()) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + Worker( + client, + task_queue=f"replay-metrics-{uuid.uuid4()}", + activities=[replay_metrics_get_weather], + workflows=[ReplayMetricsAgent], + plugins=[GoogleAdkPlugin()], + ) + warned = [ + w for w in recorded if "MeterProvider is not replay-safe" in str(w.message) + ] + assert len(warned) == 1 + assert warned[0].category is UserWarning + assert warned[0].filename == __file__ diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py index 23af2a023..4c300ff31 100644 --- a/tests/contrib/opentelemetry/test_meter_provider.py +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -1,9 +1,14 @@ """Unit tests for ReplaySafeMeterProvider outside workflows.""" +import subprocess +import sys +import textwrap from collections.abc import Iterable +from opentelemetry.context import Context from opentelemetry.metrics import ( CallbackOptions, + Counter, Histogram, Meter, MeterProvider, @@ -102,15 +107,36 @@ def get_meter( def test_replay_safe_meter_provider_supports_older_otel_signatures(): - """Newer opentelemetry-api parameters (get_meter attributes, 1.26; - create_histogram explicit_bucket_boundaries_advisory, 1.30) must only be - forwarded when set, so providers with older signatures keep working.""" + """Newer opentelemetry-api parameters (get_meter attributes, 1.26; sync + instrument context, 1.28; create_histogram + explicit_bucket_boundaries_advisory, 1.30) must only be forwarded when + set, so providers with older signatures keep working.""" + + class Pre128Counter(Counter): + def __init__(self) -> None: + self.calls: list[tuple[int | float, Attributes | None]] = [] + + def add( # type: ignore[override] + self, + amount: int | float, + attributes: Attributes | None = None, + ) -> None: + self.calls.append((amount, attributes)) class Pre130Meter(NoOpMeter): def __init__(self) -> None: super().__init__("pre-1.30-meter") + self.counter = Pre128Counter() self.histogram_calls: list[tuple[str, str, str]] = [] + def create_counter( # type: ignore[override] + self, + name: str, + unit: str = "", + description: str = "", + ) -> Counter: + return self.counter + def create_histogram( # type: ignore[override] self, name: str, @@ -138,9 +164,60 @@ def get_meter( # type: ignore[override] provider = ReplaySafeMeterProvider(inner_provider) meter = provider.get_meter("test-meter") meter.create_histogram("histogram").record(1) + meter.create_counter("counter").add(2, {"attr": "val"}) assert inner_provider.get_meter_calls == [("test-meter", None, None)] assert inner_provider.meter.histogram_calls == [("histogram", "", "")] + assert inner_provider.meter.counter.calls == [(2, {"attr": "val"})] + + +def test_replay_safe_meter_provider_forwards_context_when_set(): + class RecordingCounter(Counter): + def __init__(self) -> None: + self.calls: list[tuple[int | float, Attributes | None, Context | None]] = [] + + def add( + self, + amount: int | float, + attributes: Attributes | None = None, + context: Context | None = None, + ) -> None: + self.calls.append((amount, attributes, context)) + + class RecordingMeter(NoOpMeter): + def __init__(self) -> None: + super().__init__("recording-meter") + self.counter = RecordingCounter() + + def create_counter( # type: ignore[override] + self, + name: str, + unit: str = "", + description: str = "", + ) -> Counter: + return self.counter + + class RecordingProvider(NoOpMeterProvider): + def __init__(self) -> None: + self.meter = RecordingMeter() + + def get_meter( # type: ignore[override] + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes | None = None, + ) -> Meter: + return self.meter + + inner_provider = RecordingProvider() + provider = ReplaySafeMeterProvider(inner_provider) + context = Context() + provider.get_meter("test-meter").create_counter("counter").add( + 3, {"attr": "val"}, context + ) + + assert inner_provider.meter.counter.calls == [(3, {"attr": "val"}, context)] def test_replay_safe_meter_provider_delegates_other_attributes(): @@ -148,3 +225,71 @@ def test_replay_safe_meter_provider_delegates_other_attributes(): provider = ReplaySafeMeterProvider(inner_provider) assert provider.force_flush() provider.shutdown() + + +def _run_in_subprocess(code: str) -> None: + # Import-time behavior must be tested in a fresh interpreter so the + # simulated old opentelemetry-api is seen before temporalio imports it and + # no module state leaks into other tests. + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" + + +def test_module_imports_without_sync_gauge(): + """opentelemetry-api 1.12 through 1.22 has no opentelemetry.metrics._Gauge; + the module must still import and wrap the other instruments.""" + _run_in_subprocess( + """ + import opentelemetry.metrics + from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + del opentelemetry.metrics._Gauge + + from temporalio.contrib.opentelemetry import ReplaySafeMeterProvider + + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + provider.get_meter("test-meter").create_counter("counter").add(1) + + data = reader.get_metrics_data() + assert data is not None + metrics = [ + metric.name + for rm in data.resource_metrics + for sm in rm.scope_metrics + for metric in sm.metrics + ] + assert metrics == ["counter"], metrics + """ + ) + + +def test_tracing_importable_without_metrics_api(): + """opentelemetry-api < 1.12 has no opentelemetry.metrics module at all; + tracing users must be unaffected and ReplaySafeMeterProvider must raise an + actionable error on access. Blocking opentelemetry.metrics itself would + also break the modern opentelemetry-sdk trace module installed here, so + simulate by failing the guarded submodule import.""" + _run_in_subprocess( + """ + import sys + + sys.modules["temporalio.contrib.opentelemetry._meter_provider"] = None + + import temporalio.contrib.opentelemetry as otel_contrib + + assert otel_contrib.ReplaySafeTracerProvider is not None + assert otel_contrib.create_tracer_provider is not None + try: + otel_contrib.ReplaySafeMeterProvider + except ImportError as err: + assert "opentelemetry-api >= 1.12" in str(err), str(err) + else: + raise AssertionError("expected ImportError accessing ReplaySafeMeterProvider") + """ + ) From 553ce6cafbbac7eec0506c39ca982726523a7034 Mon Sep 17 00:00:00 2001 From: DABH Date: Sat, 1 Aug 2026 03:27:37 -0500 Subject: [PATCH 05/24] Add replay-safe OpenTelemetry logger provider ADK emits gen_ai.* log events from workflow code through the global logger provider, so replays duplicate them like metrics and spans. ReplaySafeLoggerProvider wraps a LoggerProvider and drops Logger.emit during replay. The logs API only exists in opentelemetry-api >= 1.15, so the import is guarded like the meter provider's; emit forwards arguments verbatim since its signature changed in 1.38. --- temporalio/contrib/opentelemetry/README.md | 24 +++ temporalio/contrib/opentelemetry/__init__.py | 23 ++- .../contrib/opentelemetry/_logger_provider.py | 110 ++++++++++++ .../opentelemetry/test_logger_provider.py | 169 ++++++++++++++++++ 4 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 temporalio/contrib/opentelemetry/_logger_provider.py create mode 100644 tests/contrib/opentelemetry/test_logger_provider.py diff --git a/temporalio/contrib/opentelemetry/README.md b/temporalio/contrib/opentelemetry/README.md index f436b7434..0010cf73d 100644 --- a/temporalio/contrib/opentelemetry/README.md +++ b/temporalio/contrib/opentelemetry/README.md @@ -267,6 +267,30 @@ retried workflow task re-executes live and can record again. Observable (asynchronous) instruments and recordings made outside workflows pass through untouched. +## Replay-Safe Log Events + +Libraries may also emit OpenTelemetry log records through the process-global +logger provider from workflow code (e.g. Google ADK's `gen_ai.*` events), +which duplicate on every replay the same way. `ReplaySafeLoggerProvider` +wraps your logger provider so records emitted from workflow code are dropped +during replay: + +```python +import opentelemetry._logs +from opentelemetry.sdk._logs import LoggerProvider +from temporalio.contrib.opentelemetry import ReplaySafeLoggerProvider + +# set_logger_provider only takes effect once per process, so this wrapper +# must be the first and only global logger provider set, installed before +# any library emits log records. +opentelemetry._logs.set_logger_provider( + ReplaySafeLoggerProvider(my_logger_provider) +) +``` + +Emissions are first-execution-only: a retried workflow task re-executes live +and can emit again. Emissions outside workflows pass through untouched. + ## Best Practices 1. **Register on Client**: Always register plugins/interceptors on the client, not the worker, to ensure proper context propagation diff --git a/temporalio/contrib/opentelemetry/__init__.py b/temporalio/contrib/opentelemetry/__init__.py index 25dbb9a62..c65518f42 100644 --- a/temporalio/contrib/opentelemetry/__init__.py +++ b/temporalio/contrib/opentelemetry/__init__.py @@ -23,6 +23,17 @@ # ReplaySafeMeterProvider is actually accessed (see __getattr__ below). _meter_provider_import_error = err +_logger_provider_import_error: ImportError | None = None +try: + from temporalio.contrib.opentelemetry._logger_provider import ( + ReplaySafeLoggerProvider, + ) +except ImportError as err: + # opentelemetry-api < 1.15 has no opentelemetry._logs module. Keep the + # tracing integration importable and raise a clear error only when + # ReplaySafeLoggerProvider is actually accessed (see __getattr__ below). + _logger_provider_import_error = err + from temporalio.contrib.opentelemetry._otel_interceptor import OpenTelemetryInterceptor from temporalio.contrib.opentelemetry._plugin import OpenTelemetryPlugin from temporalio.contrib.opentelemetry._tracer_provider import ( @@ -35,6 +46,7 @@ "TracingWorkflowInboundInterceptor", "OpenTelemetryInterceptor", "OpenTelemetryPlugin", + "ReplaySafeLoggerProvider", "ReplaySafeMeterProvider", "ReplaySafeTracerProvider", "create_tracer_provider", @@ -42,8 +54,9 @@ def __getattr__(name: str) -> Any: - # Only reachable for ReplaySafeMeterProvider when the guarded import above - # failed; otherwise the module attribute exists and this is never called. + # Only reachable for the replay-safe providers when their guarded imports + # above failed; otherwise the module attributes exist and this is never + # called. if name == "ReplaySafeMeterProvider": raise ImportError( "ReplaySafeMeterProvider requires the OpenTelemetry metrics API " @@ -51,4 +64,10 @@ def __getattr__(name: str) -> Any: "version does not provide. Install opentelemetry-api >= 1.12 " "(>= 1.23 for synchronous gauge support)." ) from _meter_provider_import_error + if name == "ReplaySafeLoggerProvider": + raise ImportError( + "ReplaySafeLoggerProvider requires the OpenTelemetry logs API " + "(opentelemetry._logs), which the installed opentelemetry-api " + "version does not provide. Install opentelemetry-api >= 1.15." + ) from _logger_provider_import_error raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/temporalio/contrib/opentelemetry/_logger_provider.py b/temporalio/contrib/opentelemetry/_logger_provider.py new file mode 100644 index 000000000..e495f7480 --- /dev/null +++ b/temporalio/contrib/opentelemetry/_logger_provider.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from opentelemetry._logs import Logger, LoggerProvider + +from temporalio import workflow + +if TYPE_CHECKING: + from opentelemetry.util.types import _ExtendedAttributes + + +def _skip_emitting() -> bool: + # in_workflow() must be evaluated first: is_replaying() requires an active + # workflow context. + return workflow.in_workflow() and workflow.unsafe.is_replaying() + + +class _ReplaySafeLogger(Logger): + def __init__(self, logger: Logger) -> None: + self._logger = logger + + def __getattr__(self, name: str) -> object: + return getattr(self._logger, name) + + def emit(self, *args: Any, **kwargs: Any) -> None: + # emit's signature differs across the supported opentelemetry-api + # range (a single positional LogRecord through 1.37, keyword fields + # from 1.38), so forward arguments verbatim rather than pinning one + # shape; the wrapped logger comes from the same installed API. + if _skip_emitting(): + # Skip emitting log records during workflow replay to avoid duplicate telemetry + return + self._logger.emit(*args, **kwargs) + + +class ReplaySafeLoggerProvider(LoggerProvider): + """A logger provider that is safe for use during workflow replay. + + .. warning:: + This class is experimental and may change in future versions. + Use with caution in production environments. + + This logger provider wraps an OpenTelemetry LoggerProvider and drops log + records emitted (``Logger.emit()``) from workflow code while the workflow + is replaying. Without this, libraries that emit OpenTelemetry log records + from workflow code (e.g. ``google-adk``'s ``gen_ai.*`` events) re-emit + every record on each replay, duplicating telemetry. + + Emissions are therefore first-execution-only: a workflow task retry + re-executes live and can emit again. Emissions outside workflows are + unaffected. + + Install this as the process-global logger provider before any library + (e.g. ``google-adk``) obtains loggers:: + + opentelemetry._logs.set_logger_provider( + ReplaySafeLoggerProvider(my_logger_provider) + ) + + On opentelemetry-api 1.23+ proxy loggers late-bind, so calling + ``set_logger_provider`` after such libraries are imported still routes + their loggers through this wrapper; on 1.15 through 1.22 loggers obtained + before ``set_logger_provider`` stay no-op, so install this provider first. + ``set_logger_provider`` only takes effect once per process, so this + wrapper must be the one and only global logger provider ever set. + """ + + def __init__(self, logger_provider: LoggerProvider) -> None: + """Initialize the replay-safe logger provider. + + Args: + logger_provider: The underlying OpenTelemetry LoggerProvider to wrap. + """ + self._logger_provider = logger_provider + + def __getattr__(self, name: str) -> Any: + """Delegate all other attributes (e.g. ``shutdown``, ``force_flush``) + to the underlying logger provider. + """ + return getattr(self._logger_provider, name) + + def get_logger( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: _ExtendedAttributes | None = None, + ) -> Logger: + """Get a replay-safe logger from the underlying provider. + + Args: + name: The name of the instrumenting module. + version: The version string of the instrumenting library. + schema_url: The schema URL for the logger. + attributes: Instrumentation scope attributes for the logger. + + Returns: + A replay-safe logger instance. + """ + # Forward attributes only when set: the parameter was added in + # opentelemetry 1.26 and passing it to older providers raises + # TypeError. + if attributes is None: + inner = self._logger_provider.get_logger(name, version, schema_url) + else: + inner = self._logger_provider.get_logger( + name, version, schema_url, attributes + ) + return _ReplaySafeLogger(inner) diff --git a/tests/contrib/opentelemetry/test_logger_provider.py b/tests/contrib/opentelemetry/test_logger_provider.py new file mode 100644 index 000000000..ffcfaec9b --- /dev/null +++ b/tests/contrib/opentelemetry/test_logger_provider.py @@ -0,0 +1,169 @@ +"""Unit tests for ReplaySafeLoggerProvider outside workflows.""" + +import subprocess +import sys +import textwrap +from typing import Any + +from opentelemetry._logs import ( + Logger, + LoggerProvider, + LogRecord, + NoOpLogger, + NoOpLoggerProvider, +) +from opentelemetry.sdk._logs import LoggerProvider as SdkLoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, +) +from opentelemetry.util.types import _ExtendedAttributes + +from temporalio.contrib.opentelemetry import ReplaySafeLoggerProvider + + +def _sdk_provider() -> tuple[ReplaySafeLoggerProvider, InMemoryLogRecordExporter]: + exporter = InMemoryLogRecordExporter() + inner = SdkLoggerProvider() + inner.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + return ReplaySafeLoggerProvider(inner), exporter + + +def test_replay_safe_logger_provider_emit_passes_through_outside_workflow(): + provider, exporter = _sdk_provider() + logger = provider.get_logger("test-logger") + + logger.emit(LogRecord(event_name="record-form", body="hello")) + logger.emit(event_name="kwargs-form", body="world", attributes={"attr": "val"}) + + records = [log.log_record for log in exporter.get_finished_logs()] + assert [(r.event_name, r.body) for r in records] == [ + ("record-form", "hello"), + ("kwargs-form", "world"), + ] + assert records[1].attributes and dict(records[1].attributes) == {"attr": "val"} + + +def test_replay_safe_logger_provider_delegates_get_logger_arguments(): + class RecordingLoggerProvider(LoggerProvider): + def __init__(self) -> None: + self.calls: list[tuple] = [] + self._inner = SdkLoggerProvider() + + def get_logger( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: _ExtendedAttributes | None = None, + ) -> Logger: + self.calls.append((name, version, schema_url, attributes)) + return self._inner.get_logger(name, version, schema_url, attributes) + + inner_provider = RecordingLoggerProvider() + provider = ReplaySafeLoggerProvider(inner_provider) + provider.get_logger( + "test-logger", + version="1.2.3", + schema_url="https://example.com/schema", + attributes={"attr": "val"}, + ) + + assert inner_provider.calls == [ + ("test-logger", "1.2.3", "https://example.com/schema", {"attr": "val"}) + ] + + +def test_replay_safe_logger_provider_supports_older_otel_signatures(): + """Newer opentelemetry-api parameters (get_logger attributes, 1.26; emit + keyword fields, 1.38) must only be forwarded when the caller passes them, + so loggers and providers with older signatures keep working.""" + + class Pre138Logger(NoOpLogger): + def __init__(self) -> None: + super().__init__("pre-1.38-logger") + self.records: list[LogRecord] = [] + + def emit(self, record: LogRecord) -> None: # type: ignore[override] + self.records.append(record) + + class Pre126LoggerProvider(NoOpLoggerProvider): + def __init__(self) -> None: + self.logger = Pre138Logger() + self.get_logger_calls: list[tuple[str, str | None, str | None]] = [] + + def get_logger( # type: ignore[override] + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + ) -> Logger: + self.get_logger_calls.append((name, version, schema_url)) + return self.logger + + inner_provider = Pre126LoggerProvider() + provider = ReplaySafeLoggerProvider(inner_provider) + record = LogRecord(event_name="event", body="hello") + provider.get_logger("test-logger").emit(record) + + assert inner_provider.get_logger_calls == [("test-logger", None, None)] + assert inner_provider.logger.records == [record] + + +def test_replay_safe_logger_provider_delegates_other_attributes(): + provider, _ = _sdk_provider() + assert provider.force_flush() + provider.shutdown() + + +def test_replay_safe_logger_provider_delegates_other_logger_attributes(): + class AttributedLogger(NoOpLogger): + def __init__(self) -> None: + super().__init__("attributed-logger") + self.custom = "custom-value" + + class AttributedLoggerProvider(NoOpLoggerProvider): + def get_logger(self, *args: Any, **kwargs: Any) -> Logger: # type: ignore[override] + return AttributedLogger() + + logger = ReplaySafeLoggerProvider(AttributedLoggerProvider()).get_logger("test") + assert logger.custom == "custom-value" # type: ignore[attr-defined] + + +def _run_in_subprocess(code: str) -> None: + # Import-time behavior must be tested in a fresh interpreter so the + # simulated old opentelemetry-api is seen before temporalio imports it and + # no module state leaks into other tests. + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" + + +def test_tracing_importable_without_logs_api(): + """opentelemetry-api < 1.15 has no opentelemetry._logs module at all; + tracing and metrics users must be unaffected and ReplaySafeLoggerProvider + must raise an actionable error on access. Blocking opentelemetry._logs + itself would also break the modern opentelemetry-sdk installed here, so + simulate by failing the guarded submodule import.""" + _run_in_subprocess( + """ + import sys + + sys.modules["temporalio.contrib.opentelemetry._logger_provider"] = None + + import temporalio.contrib.opentelemetry as otel_contrib + + assert otel_contrib.ReplaySafeTracerProvider is not None + assert otel_contrib.ReplaySafeMeterProvider is not None + assert otel_contrib.create_tracer_provider is not None + try: + otel_contrib.ReplaySafeLoggerProvider + except ImportError as err: + assert "opentelemetry-api >= 1.15" in str(err), str(err) + else: + raise AssertionError("expected ImportError accessing ReplaySafeLoggerProvider") + """ + ) From 891b68b2d3f8d283d1435635963ffe571e9f0061 Mon Sep 17 00:00:00 2001 From: DABH Date: Sat, 1 Aug 2026 03:27:47 -0500 Subject: [PATCH 06/24] Extend ADK replay-safety warnings to logs and replayer Validate the global logger provider alongside meter and tracer, and run the validation from configure_replayer too since Replayer replays are exactly where unsafe providers re-record telemetry. Warn only on providers positively identified as replay-unsafe (OTel SDK providers used directly) so custom wrappers around replay-safe providers no longer trigger false positives, and compute the warning stacklevel dynamically so attribution survives wrapping plugins. Add log-event replay regression tests mirroring the metrics ones. --- CHANGELOG.md | 18 +- .../contrib/google_adk_agents/README.md | 31 ++- .../contrib/google_adk_agents/_plugin.py | 121 +++++++-- tests/conftest.py | 17 ++ .../google_adk_agents/test_replay_metrics.py | 253 ++++++++++++++++-- 5 files changed, 372 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7854ccbfe..892d01758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,15 +20,17 @@ to include examples, links to docs, or any other relevant information. ### Added -- Added experimental `temporalio.contrib.opentelemetry.ReplaySafeMeterProvider`, a wrapping - OpenTelemetry `MeterProvider` that drops synchronous instrument recordings made from workflow - code during replay. Install it as the process-global meter provider when libraries record - OpenTelemetry metrics from workflow code (e.g. Google ADK) so that workflow replays (cache - eviction, worker restarts, redeploys) do not re-record metrics; recordings are - first-execution-only, matching `temporalio.workflow.metric_meter()`. +- Added experimental `temporalio.contrib.opentelemetry.ReplaySafeMeterProvider` and + `ReplaySafeLoggerProvider`, wrapping OpenTelemetry providers that drop synchronous instrument + recordings and emitted log records made from workflow code during replay. Install them as the + process-global providers when libraries record OpenTelemetry metrics or emit log events from + workflow code (e.g. Google ADK) so that workflow replays (cache eviction, worker restarts, + redeploys) do not duplicate telemetry; recordings are first-execution-only, matching + `temporalio.workflow.metric_meter()`. `temporalio.contrib.opentelemetry.ReplaySafeTracerProvider` is now also exported. - `GoogleAdkPlugin` now warns at worker configuration time when the global OpenTelemetry meter - or tracer provider is not replay-safe. + `GoogleAdkPlugin` now warns at worker and replayer configuration time when the global + OpenTelemetry meter, tracer, or logger provider is positively identified as not replay-safe + (an OpenTelemetry SDK provider used directly). ### Changed diff --git a/temporalio/contrib/google_adk_agents/README.md b/temporalio/contrib/google_adk_agents/README.md index 54c55938a..d2f540956 100644 --- a/temporalio/contrib/google_adk_agents/README.md +++ b/temporalio/contrib/google_adk_agents/README.md @@ -202,13 +202,14 @@ agent = Agent( ## Telemetry and Workflow Replay ADK records OpenTelemetry metrics (scope `gcp.vertex.agent`, e.g. -`gen_ai.client.token.usage`) and spans through the process-global -OpenTelemetry providers from code that runs inside the workflow. Workflow -code re-executes on every replay, so with a plain global provider each replay -re-records all of that telemetry even though no model or tool actually ran -again — for example, 1 real execution followed by 3 replays yields 4x the -observations on every instrument. Replays happen routinely in production: -workflow cache eviction, worker restarts, redeploys, or running with +`gen_ai.client.token.usage`), spans, and log events (e.g. `gen_ai.choice`) +through the process-global OpenTelemetry providers from code that runs +inside the workflow. Workflow code re-executes on every replay, so with a +plain global provider each replay re-records all of that telemetry even +though no model or tool actually ran again — for example, 1 real execution +followed by 3 replays yields 4x the observations on every instrument and 4 +copies of every log event. Replays happen routinely in production: workflow +cache eviction, worker restarts, redeploys, or running with `max_cached_workflows=0`. To avoid this, install Temporal's replay-safe providers as the global @@ -216,19 +217,23 @@ OpenTelemetry providers. They pass recordings through on first execution and drop them during replay: ```python +import opentelemetry._logs import opentelemetry.metrics import opentelemetry.trace +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.trace.export import BatchSpanProcessor from temporalio.contrib.opentelemetry import ( + ReplaySafeLoggerProvider, ReplaySafeMeterProvider, create_tracer_provider, ) -# set_meter_provider/set_tracer_provider only take effect once per process, -# so these wrappers must be the first and only global providers set. +# The global set_*_provider functions only take effect once per process, so +# these wrappers must be the first and only global providers set. opentelemetry.metrics.set_meter_provider( ReplaySafeMeterProvider( MeterProvider(metric_readers=[PeriodicExportingMetricReader(my_exporter)]) @@ -237,10 +242,14 @@ opentelemetry.metrics.set_meter_provider( tracer_provider = create_tracer_provider() tracer_provider.add_span_processor(BatchSpanProcessor(my_span_exporter)) opentelemetry.trace.set_tracer_provider(tracer_provider) +logger_provider = LoggerProvider() +logger_provider.add_log_record_processor(BatchLogRecordProcessor(my_log_exporter)) +opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(logger_provider)) ``` -`GoogleAdkPlugin` warns at worker configuration time when a global provider -is installed that is not replay-safe. +`GoogleAdkPlugin` warns at worker and replayer configuration time when a +global provider is installed that it can positively identify as not +replay-safe (an OpenTelemetry SDK provider used directly). Recordings are first-execution-only, matching `temporalio.workflow.metric_meter()`: a retried workflow task re-executes diff --git a/temporalio/contrib/google_adk_agents/_plugin.py b/temporalio/contrib/google_adk_agents/_plugin.py index 44f8d073e..2f1d34b71 100644 --- a/temporalio/contrib/google_adk_agents/_plugin.py +++ b/temporalio/contrib/google_adk_agents/_plugin.py @@ -1,17 +1,28 @@ from __future__ import annotations import dataclasses +import sys import time import uuid import warnings from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager +from types import FrameType from typing import Any +import opentelemetry._logs import opentelemetry.metrics import opentelemetry.trace +from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.trace import NoOpTracerProvider, ProxyTracerProvider +from opentelemetry.sdk._logs import LoggerProvider as SdkLoggerProvider +from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider +from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider +from opentelemetry.trace import ( + NoOpTracerProvider, + ProxyTracerProvider, + TracerProvider, +) from temporalio import workflow from temporalio.contrib.google_adk_agents._mcp import TemporalMcpToolSetProvider @@ -20,6 +31,7 @@ invoke_model_streaming, ) from temporalio.contrib.opentelemetry import ( + ReplaySafeLoggerProvider, ReplaySafeMeterProvider, ReplaySafeTracerProvider, ) @@ -30,20 +42,21 @@ from temporalio.converter import DataConverter, DefaultPayloadConverter from temporalio.plugin import SimplePlugin from temporalio.worker import ( + ReplayerConfig, WorkerConfig, WorkflowRunner, ) from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner +# Each classifier below returns True when the provider is replay-safe +# (including providers that drop all recordings), False when positively +# identified as replay-unsafe (an OpenTelemetry SDK provider), and None when +# it cannot be classified. Unknown provider types (e.g. a custom provider +# delegating to a replay-safe one) must not warn: a false positive is worse +# than a missed warning. -def _meter_provider_replay_safe(provider: MeterProvider) -> bool | None: - """Classify the global meter provider for replay safety. - Returns True when replay-safe (including providers that drop all - recordings), False when positively identified as replay-unsafe, and None - when it cannot be classified, in which case no warning should be issued - since a false positive is worse than a missed warning. - """ +def _meter_provider_replay_safe(provider: MeterProvider) -> bool | None: if isinstance(provider, (ReplaySafeMeterProvider, NoOpMeterProvider)): return True try: @@ -52,22 +65,61 @@ def _meter_provider_replay_safe(provider: MeterProvider) -> bool | None: # so a moved or removed symbol cannot break module import; it is # present in opentelemetry-api 1.12 through at least 1.42. from opentelemetry.metrics._internal import _ProxyMeterProvider + + if isinstance(provider, _ProxyMeterProvider): + return True + except ImportError: + pass + return False if isinstance(provider, SdkMeterProvider) else None + + +def _tracer_provider_replay_safe(provider: TracerProvider) -> bool | None: + if isinstance( + provider, + (ReplaySafeTracerProvider, NoOpTracerProvider, ProxyTracerProvider), + ): + return True + return False if isinstance(provider, SdkTracerProvider) else None + + +def _logger_provider_replay_safe(provider: LoggerProvider) -> bool | None: + if isinstance(provider, (ReplaySafeLoggerProvider, NoOpLoggerProvider)): + return True + try: + # The proxy (unset) logger provider has no public counterpart either; + # present in opentelemetry-api 1.23 through at least 1.42. + from opentelemetry._logs._internal import ProxyLoggerProvider + + if isinstance(provider, ProxyLoggerProvider): + return True except ImportError: - return None - return isinstance(provider, _ProxyMeterProvider) + pass + return False if isinstance(provider, SdkLoggerProvider) else None + + +def _stacklevel_outside_temporalio() -> int: + # Attribute provider warnings to the nearest frame outside temporalio, + # e.g. the user's Worker(...)/Replayer(...) call or a user plugin that + # delegates here, however many plugin frames sit in between. + level = 1 + frame: FrameType | None = sys._getframe(1) + while frame is not None: + module = frame.f_globals.get("__name__", "") + if module != "temporalio" and not module.startswith("temporalio."): + return level + frame = frame.f_back + level += 1 + return 1 def _warn_if_global_otel_providers_not_replay_safe() -> None: - # ADK records metrics and spans through the process-global OpenTelemetry - # providers from code that runs workflow-side, so a non-replay-safe global - # provider re-emits that telemetry on every workflow replay. Unset (proxy) - # and no-op providers drop recordings and are fine. - # - # stacklevel=4 attributes the warnings to the user's Worker(...) call: - # warn -> this helper -> GoogleAdkPlugin.configure_worker -> - # Worker.__init__ -> user code. - meter_provider = opentelemetry.metrics.get_meter_provider() - if _meter_provider_replay_safe(meter_provider) is False: + # ADK records metrics, spans, and log events through the process-global + # OpenTelemetry providers from code that runs workflow-side, so a + # non-replay-safe global provider re-emits that telemetry on every + # workflow replay. Unset (proxy) and no-op providers drop recordings and + # are fine. + stacklevel = _stacklevel_outside_temporalio() + if _meter_provider_replay_safe(opentelemetry.metrics.get_meter_provider()) is False: warnings.warn( "The global OpenTelemetry MeterProvider is not replay-safe: Google ADK " "records metrics from workflow code, so every workflow replay will " @@ -76,13 +128,9 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: "the first and only global provider set: " "opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(provider))", UserWarning, - stacklevel=4, + stacklevel=stacklevel, ) - tracer_provider = opentelemetry.trace.get_tracer_provider() - if not isinstance( - tracer_provider, - (ReplaySafeTracerProvider, NoOpTracerProvider, ProxyTracerProvider), - ): + if _tracer_provider_replay_safe(opentelemetry.trace.get_tracer_provider()) is False: warnings.warn( "The global OpenTelemetry TracerProvider is not replay-safe: Google ADK " "creates spans from workflow code, so every workflow replay will " @@ -90,7 +138,18 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: "opentelemetry.trace.set_tracer_provider(" "temporalio.contrib.opentelemetry.create_tracer_provider())", UserWarning, - stacklevel=4, + stacklevel=stacklevel, + ) + if _logger_provider_replay_safe(opentelemetry._logs.get_logger_provider()) is False: + warnings.warn( + "The global OpenTelemetry LoggerProvider is not replay-safe: Google ADK " + "emits log events (e.g. gen_ai.choice) from workflow code, so every " + "workflow replay will re-emit them. Wrap your provider in " + "temporalio.contrib.opentelemetry.ReplaySafeLoggerProvider and make it " + "the first and only global provider set: " + "opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(provider))", + UserWarning, + stacklevel=stacklevel, ) @@ -194,6 +253,14 @@ def configure_worker(self, config: WorkerConfig) -> WorkerConfig: _warn_if_global_otel_providers_not_replay_safe() return super().configure_worker(config) + def configure_replayer(self, config: ReplayerConfig) -> ReplayerConfig: + """See base class. Also warns when the global OpenTelemetry providers + are not replay-safe, since every replayed workflow would re-emit ADK + telemetry. + """ + _warn_if_global_otel_providers_not_replay_safe() + return super().configure_replayer(config) + def _configure_data_converter( self, converter: DataConverter | None ) -> DataConverter: diff --git a/tests/conftest.py b/tests/conftest.py index 1bc3c37ea..ebdc52a67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import sys from collections.abc import AsyncGenerator, Iterator +import opentelemetry._logs._internal import opentelemetry.metrics._internal import opentelemetry.trace import pytest @@ -285,3 +286,19 @@ def reset_otel_meter_provider(): yield opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() opentelemetry.metrics._internal._METER_PROVIDER = None + + +@pytest.fixture +def reset_otel_logger_provider(): + """Reset global OpenTelemetry logger provider state around tests. + + Unlike proxy meters, proxy loggers cache their real logger on first use + and never rebind, even across a later set_logger_provider call. Tests + exercising a library's module-level logger must clear that cache + themselves (e.g. Google ADK's telemetry logger). + """ + opentelemetry._logs._internal._LOGGER_PROVIDER_SET_ONCE = Once() + opentelemetry._logs._internal._LOGGER_PROVIDER = None + yield + opentelemetry._logs._internal._LOGGER_PROVIDER_SET_ONCE = Once() + opentelemetry._logs._internal._LOGGER_PROVIDER = None diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index 5dba5dfb3..6749e1671 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -1,20 +1,24 @@ -"""Tests for replay-safe handling of ADK's OpenTelemetry metrics. - -Google ADK records token-usage/latency metrics through the process-global -OpenTelemetry meter (scope "gcp.vertex.agent") from code that runs -workflow-side under the Temporal adapter. Without gating, 1 real execution + -N replays produces (1 + N) observations per instrument, while the activity -(the model / tool call) runs exactly once. Installing -ReplaySafeMeterProvider as the global meter provider suppresses the replay +"""Tests for replay-safe handling of ADK's OpenTelemetry metrics and log events. + +Google ADK records token-usage/latency metrics and emits gen_ai.* log events +through the process-global OpenTelemetry providers (scope "gcp.vertex.agent") +from code that runs workflow-side under the Temporal adapter. Without gating, +1 real execution + N replays produces (1 + N) observations per instrument and +(1 + N) copies of every log event, while the activity (the model / tool call) +runs exactly once. Installing ReplaySafeMeterProvider / +ReplaySafeLoggerProvider as the global providers suppresses the replay recordings while leaving first-execution recordings intact. """ +import inspect import sys import uuid import warnings from collections.abc import AsyncGenerator from datetime import timedelta +import google.adk.telemetry.tracing +import opentelemetry._logs import opentelemetry.metrics import pytest from google.adk import Agent @@ -24,18 +28,29 @@ from google.adk.runners import InMemoryRunner from google.adk.utils.context_utils import Aclosing from google.genai import types -from opentelemetry.metrics import NoOpMeterProvider +from opentelemetry._logs import NoOpLoggerProvider +from opentelemetry._logs._internal import ProxyLoggerProvider +from opentelemetry.metrics import Meter, NoOpMeterProvider +from opentelemetry.metrics import MeterProvider as ApiMeterProvider from opentelemetry.metrics._internal import _ProxyMeterProvider +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import TracerProvider as ApiTracerProvider from opentelemetry.trace import set_tracer_provider +from opentelemetry.util.types import Attributes import temporalio.contrib.google_adk_agents.workflow from temporalio import activity, workflow from temporalio.client import Client, WorkflowHistory from temporalio.contrib.google_adk_agents import GoogleAdkPlugin, TemporalModel from temporalio.contrib.opentelemetry import ( + ReplaySafeLoggerProvider, ReplaySafeMeterProvider, create_tracer_provider, ) @@ -56,6 +71,15 @@ "gen_ai.client.token.usage": 4, } +# Log events per model call: one gen_ai.system.message, one +# gen_ai.user.message per request content (1 for the first call; 3 for the +# second: prompt + tool call + tool response), one gen_ai.choice per result. +EXPECTED_LOG_BASELINE = { + "gen_ai.system.message": 2, + "gen_ai.user.message": 4, + "gen_ai.choice": 2, +} + # Counts real (worker-side) activity executions; replays must not add to it. activity_executions = 0 @@ -173,6 +197,40 @@ def adk_metric_counts(reader: InMemoryMetricReader) -> dict[str, int]: return counts +def adk_log_event_counts(exporter: InMemoryLogRecordExporter) -> dict[str, int]: + """Emission count per ADK log event name.""" + counts: dict[str, int] = {} + for log in exporter.get_finished_logs(): + if log.instrumentation_scope is None or ( + log.instrumentation_scope.name != ADK_METER_SCOPE + ): + continue + event_name = log.log_record.event_name + if event_name: + counts[event_name] = counts.get(event_name, 0) + 1 + return counts + + +@pytest.fixture +def reset_adk_proxy_logger(): + """Clear ADK's cached proxy logger binding around tests. + + ADK's module-level otel_logger is a proxy that caches its real logger on + first emit and never rebinds, even across a later set_logger_provider + call, so each test must clear the cache for its own provider to receive + the events. + """ + + def clear() -> None: + proxy = google.adk.telemetry.tracing.otel_logger + if hasattr(proxy, "_real_logger"): + proxy._real_logger = None # type: ignore[attr-defined] + + clear() + yield + clear() + + async def _run_once_and_replay( client: Client, num_replays: int ) -> tuple[int, WorkflowHistory]: @@ -250,6 +308,47 @@ async def test_replay_metrics_duplicate_without_replay_safe_meter_provider( } +def _in_memory_logger_provider() -> tuple[LoggerProvider, InMemoryLogRecordExporter]: + exporter = InMemoryLogRecordExporter() + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + return provider, exporter + + +async def test_replay_safe_logger_provider_suppresses_replay_log_events( + client: Client, + reset_otel_logger_provider, # type: ignore[reportUnusedParameter] + reset_adk_proxy_logger, # type: ignore[reportUnusedParameter] +): + provider, exporter = _in_memory_logger_provider() + opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(provider)) + + real_executions, _ = await _run_once_and_replay(client, num_replays=3) + + # First execution emitted exactly once (not suppressed), replays added + # zero log events, and the activity never re-executed. + assert real_executions == 1 + assert adk_log_event_counts(exporter) == EXPECTED_LOG_BASELINE + + +async def test_replay_log_events_duplicate_without_replay_safe_logger_provider( + client: Client, + reset_otel_logger_provider, # type: ignore[reportUnusedParameter] + reset_adk_proxy_logger, # type: ignore[reportUnusedParameter] +): + # Control: without the wrapper, every replay re-emits every workflow-side + # ADK log event even though nothing really re-executed. + provider, exporter = _in_memory_logger_provider() + opentelemetry._logs.set_logger_provider(provider) + + real_executions, _ = await _run_once_and_replay(client, num_replays=3) + + assert real_executions == 1 + assert adk_log_event_counts(exporter) == { + name: count * (1 + 3) for name, count in EXPECTED_LOG_BASELINE.items() + } + + def _worker_config() -> WorkerConfig: return WorkerConfig(workflow_runner=SandboxedWorkflowRunner()) @@ -270,12 +369,32 @@ def test_plugin_warns_on_non_replay_safe_tracer_provider( GoogleAdkPlugin().configure_worker(_worker_config()) +def test_plugin_warns_on_non_replay_safe_logger_provider( + reset_otel_logger_provider, # type: ignore[reportUnusedParameter] +): + opentelemetry._logs.set_logger_provider(LoggerProvider()) + with pytest.warns(UserWarning, match="LoggerProvider is not replay-safe"): + GoogleAdkPlugin().configure_worker(_worker_config()) + + +def test_plugin_warns_on_replayer_construction( + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] +): + # Replayer replays are exactly where an unsafe global provider re-records + # telemetry, so the warning must fire there too. + opentelemetry.metrics.set_meter_provider(MeterProvider()) + with pytest.warns(UserWarning, match="MeterProvider is not replay-safe"): + Replayer(workflows=[ReplayMetricsAgent], plugins=[GoogleAdkPlugin()]) + + def test_plugin_does_not_warn_with_replay_safe_providers( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] + reset_otel_logger_provider, # type: ignore[reportUnusedParameter] ): opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(MeterProvider())) set_tracer_provider(create_tracer_provider()) + opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(LoggerProvider())) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") GoogleAdkPlugin().configure_worker(_worker_config()) @@ -285,6 +404,7 @@ def test_plugin_does_not_warn_with_replay_safe_providers( def test_plugin_does_not_warn_with_unset_providers( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] + reset_otel_logger_provider, # type: ignore[reportUnusedParameter] ): with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") @@ -292,6 +412,22 @@ def test_plugin_does_not_warn_with_unset_providers( assert not [w for w in recorded if "replay-safe" in str(w.message)] +class _DelegatingMeterProvider(ApiMeterProvider): + """Unknown custom provider delegating to a replay-safe one.""" + + def __init__(self) -> None: + self._inner = ReplaySafeMeterProvider(MeterProvider()) + + def get_meter( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes | None = None, + ) -> Meter: + return self._inner.get_meter(name, version, schema_url) + + def test_meter_provider_replay_safety_classification(monkeypatch: pytest.MonkeyPatch): from temporalio.contrib.google_adk_agents._plugin import ( _meter_provider_replay_safe, @@ -301,30 +437,68 @@ def test_meter_provider_replay_safety_classification(monkeypatch: pytest.MonkeyP assert _meter_provider_replay_safe(NoOpMeterProvider()) is True assert _meter_provider_replay_safe(_ProxyMeterProvider()) is True assert _meter_provider_replay_safe(MeterProvider()) is False + # Unknown provider types (e.g. a custom provider delegating to a + # replay-safe one) are unclassifiable and must not warn. + assert _meter_provider_replay_safe(_DelegatingMeterProvider()) is None - # When the private proxy class cannot be imported, providers other than - # the replay-safe/no-op ones are unclassifiable and must not warn. + # SDK providers stay positively identified even when the private proxy + # class cannot be imported; the proxy instance becomes unclassifiable. monkeypatch.setitem(sys.modules, "opentelemetry.metrics._internal", None) - assert _meter_provider_replay_safe(MeterProvider()) is None + assert _meter_provider_replay_safe(MeterProvider()) is False + assert _meter_provider_replay_safe(_ProxyMeterProvider()) is None assert _meter_provider_replay_safe(NoOpMeterProvider()) is True -def test_plugin_does_not_warn_when_proxy_detection_unavailable( - monkeypatch: pytest.MonkeyPatch, +def test_tracer_provider_replay_safety_classification(): + from temporalio.contrib.google_adk_agents._plugin import ( + _tracer_provider_replay_safe, + ) + + class DelegatingTracerProvider(ApiTracerProvider): + def __init__(self) -> None: + self._inner = create_tracer_provider() + + def get_tracer(self, *args, **kwargs): + return self._inner.get_tracer(*args, **kwargs) + + assert _tracer_provider_replay_safe(create_tracer_provider()) is True + assert _tracer_provider_replay_safe(TracerProvider()) is False + assert _tracer_provider_replay_safe(DelegatingTracerProvider()) is None + + +def test_logger_provider_replay_safety_classification(): + from temporalio.contrib.google_adk_agents._plugin import ( + _logger_provider_replay_safe, + ) + + class DelegatingLoggerProvider(opentelemetry._logs.LoggerProvider): + def __init__(self) -> None: + self._inner = ReplaySafeLoggerProvider(LoggerProvider()) + + def get_logger(self, *args, **kwargs): + return self._inner.get_logger(*args, **kwargs) + + assert _logger_provider_replay_safe(ReplaySafeLoggerProvider(LoggerProvider())) is ( + True + ) + assert _logger_provider_replay_safe(NoOpLoggerProvider()) is True + assert _logger_provider_replay_safe(ProxyLoggerProvider()) is True + assert _logger_provider_replay_safe(LoggerProvider()) is False + assert _logger_provider_replay_safe(DelegatingLoggerProvider()) is None + + +def test_plugin_does_not_warn_on_unknown_custom_provider( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] ): - # A replay-unsafe provider is set, but with the private proxy class - # unimportable it cannot be classified, so the plugin stays silent rather - # than risking a false positive. - opentelemetry.metrics.set_meter_provider(MeterProvider()) - monkeypatch.setitem(sys.modules, "opentelemetry.metrics._internal", None) + # A custom provider delegating to a replay-safe one is a fully replay-safe + # configuration; an unclassifiable provider must not trigger a false + # positive. + opentelemetry.metrics.set_meter_provider(_DelegatingMeterProvider()) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") GoogleAdkPlugin().configure_worker(_worker_config()) - assert not [ - w for w in recorded if "MeterProvider is not replay-safe" in str(w.message) - ] + assert not [w for w in recorded if "replay-safe" in str(w.message)] async def test_plugin_warning_points_at_worker_construction( @@ -350,3 +524,38 @@ async def test_plugin_warning_points_at_worker_construction( assert len(warned) == 1 assert warned[0].category is UserWarning assert warned[0].filename == __file__ + + +class _WrappingPlugin: + """User plugin that delegates to GoogleAdkPlugin through an extra frame.""" + + def __init__(self) -> None: + self._inner = GoogleAdkPlugin() + + def configure_worker(self, config: WorkerConfig) -> WorkerConfig: + return self._inner.configure_worker(config) + + +def test_plugin_warning_points_at_wrapping_plugin_caller( + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] + reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] +): + # When another plugin wraps GoogleAdkPlugin, the warning must attribute + # to the nearest user frame (the wrapper's delegation line), not SDK + # internals or a fixed frame depth. + opentelemetry.metrics.set_meter_provider(MeterProvider()) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + _WrappingPlugin().configure_worker(_worker_config()) + warned = [ + w for w in recorded if "MeterProvider is not replay-safe" in str(w.message) + ] + assert len(warned) == 1 + assert warned[0].filename == __file__ + source_lines, start = inspect.getsourcelines(_WrappingPlugin.configure_worker) + delegation_line = start + next( + offset + for offset, line in enumerate(source_lines) + if "self._inner.configure_worker" in line + ) + assert warned[0].lineno == delegation_line From 7cec23824104dc96202058b3659c6f3546907a16 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 14:15:55 -0500 Subject: [PATCH 07/24] Require opentelemetry 1.24 for the opentelemetry extra The de facto floor of temporalio.contrib.opentelemetry is already 1.24: _tracer_provider.py imports opentelemetry.util._decorator's _agnosticcontextmanager, which was added in opentelemetry-api 1.24, so the declared 1.11.1 floor has been uninstallable in practice for this contrib regardless of the new providers. Aligning the declaration with reality lets the replay-safe providers import _Gauge (exported since 1.23) and the logs API unconditionally, removing the guarded-import machinery in the package __init__ and the subprocess-based import-absence tests. The conditional kwarg forwards stay: get_meter/get_logger attributes arrived in 1.26, synchronous-instrument context in 1.28, and create_histogram's explicit_bucket_boundaries_advisory in 1.30, all above the new floor. --- CHANGELOG.md | 5 ++ pyproject.toml | 2 +- temporalio/contrib/opentelemetry/__init__.py | 51 ++----------- .../contrib/opentelemetry/_meter_provider.py | 21 ++---- .../opentelemetry/test_logger_provider.py | 42 ----------- .../opentelemetry/test_meter_provider.py | 71 ------------------- uv.lock | 4 +- 7 files changed, 19 insertions(+), 177 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 892d01758..e02881fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ to include examples, links to docs, or any other relevant information. ### Changed +- The `opentelemetry` extra now requires `opentelemetry-api`/`opentelemetry-sdk` `>= 1.24`, + aligning the declared floor with what `temporalio.contrib.opentelemetry` already required in + practice (it has depended on an API added in `opentelemetry-api` 1.24 since the tracing + integration was introduced). + ### Deprecated ### :boom: Breaking Changes diff --git a/pyproject.toml b/pyproject.toml index 2303bdf00..05ffb1275 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ [project.optional-dependencies] grpc = ["grpcio>=1.48.2,<2"] -opentelemetry = ["opentelemetry-api>=1.11.1,<2", "opentelemetry-sdk>=1.11.1,<2"] +opentelemetry = ["opentelemetry-api>=1.24,<2", "opentelemetry-sdk>=1.24,<2"] pydantic = ["pydantic>=2.0.0,<3"] openai-agents = ["openai-agents>=0.17.5", "mcp>=1.9.4, <2"] google-adk = ["google-adk>=2.2.0,<3"] diff --git a/temporalio/contrib/opentelemetry/__init__.py b/temporalio/contrib/opentelemetry/__init__.py index c65518f42..159f7f8db 100644 --- a/temporalio/contrib/opentelemetry/__init__.py +++ b/temporalio/contrib/opentelemetry/__init__.py @@ -5,35 +5,16 @@ propagation for distributed tracing. """ -from typing import Any - from temporalio.contrib.opentelemetry._interceptor import ( TracingInterceptor, TracingWorkflowInboundInterceptor, ) - -_meter_provider_import_error: ImportError | None = None -try: - from temporalio.contrib.opentelemetry._meter_provider import ( - ReplaySafeMeterProvider, - ) -except ImportError as err: - # opentelemetry-api < 1.12 has no opentelemetry.metrics module. Keep the - # tracing integration importable and raise a clear error only when - # ReplaySafeMeterProvider is actually accessed (see __getattr__ below). - _meter_provider_import_error = err - -_logger_provider_import_error: ImportError | None = None -try: - from temporalio.contrib.opentelemetry._logger_provider import ( - ReplaySafeLoggerProvider, - ) -except ImportError as err: - # opentelemetry-api < 1.15 has no opentelemetry._logs module. Keep the - # tracing integration importable and raise a clear error only when - # ReplaySafeLoggerProvider is actually accessed (see __getattr__ below). - _logger_provider_import_error = err - +from temporalio.contrib.opentelemetry._logger_provider import ( + ReplaySafeLoggerProvider, +) +from temporalio.contrib.opentelemetry._meter_provider import ( + ReplaySafeMeterProvider, +) from temporalio.contrib.opentelemetry._otel_interceptor import OpenTelemetryInterceptor from temporalio.contrib.opentelemetry._plugin import OpenTelemetryPlugin from temporalio.contrib.opentelemetry._tracer_provider import ( @@ -51,23 +32,3 @@ "ReplaySafeTracerProvider", "create_tracer_provider", ] - - -def __getattr__(name: str) -> Any: - # Only reachable for the replay-safe providers when their guarded imports - # above failed; otherwise the module attributes exist and this is never - # called. - if name == "ReplaySafeMeterProvider": - raise ImportError( - "ReplaySafeMeterProvider requires the OpenTelemetry metrics API " - "(opentelemetry.metrics), which the installed opentelemetry-api " - "version does not provide. Install opentelemetry-api >= 1.12 " - "(>= 1.23 for synchronous gauge support)." - ) from _meter_provider_import_error - if name == "ReplaySafeLoggerProvider": - raise ImportError( - "ReplaySafeLoggerProvider requires the OpenTelemetry logs API " - "(opentelemetry._logs), which the installed opentelemetry-api " - "version does not provide. Install opentelemetry-api >= 1.15." - ) from _logger_provider_import_error - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index 944f0f56c..2cfd1186a 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -1,7 +1,10 @@ from collections.abc import Sequence -from typing import TYPE_CHECKING, Any +from typing import Any from opentelemetry.context import Context + +# _Gauge is OpenTelemetry's canonical exported name for the spec-experimental +# synchronous gauge, re-exported by opentelemetry.metrics since 1.23. from opentelemetry.metrics import ( CallbackT, Counter, @@ -12,6 +15,7 @@ ObservableGauge, ObservableUpDownCounter, UpDownCounter, + _Gauge, ) from opentelemetry.util.types import Attributes @@ -25,19 +29,6 @@ def _forward_context_kwarg(context: Context | None) -> dict[str, Any]: return {} if context is None else {"context": context} -if TYPE_CHECKING: - from opentelemetry.metrics import _Gauge -else: - try: - from opentelemetry.metrics import _Gauge - except ImportError: - # Synchronous gauges (and Meter.create_gauge) were added in - # opentelemetry-api 1.23. Fall back to a plain base class so this - # module stays importable on older versions in the supported range; - # meters there have no create_gauge to wrap anyway. - _Gauge = object - - def _skip_recording() -> bool: # in_workflow() must be evaluated first: is_replaying() requires an active # workflow context. @@ -173,8 +164,6 @@ def create_gauge( unit: str = "", description: str = "", ) -> _Gauge: - # On opentelemetry-api < 1.23 the underlying meter has no create_gauge - # and this raises AttributeError, same as calling it on the raw meter. return _ReplaySafeGauge(self._meter.create_gauge(name, unit, description)) # Observable instruments pass through unwrapped: their callbacks run on the diff --git a/tests/contrib/opentelemetry/test_logger_provider.py b/tests/contrib/opentelemetry/test_logger_provider.py index ffcfaec9b..94c2a5d0a 100644 --- a/tests/contrib/opentelemetry/test_logger_provider.py +++ b/tests/contrib/opentelemetry/test_logger_provider.py @@ -1,8 +1,5 @@ """Unit tests for ReplaySafeLoggerProvider outside workflows.""" -import subprocess -import sys -import textwrap from typing import Any from opentelemetry._logs import ( @@ -128,42 +125,3 @@ def get_logger(self, *args: Any, **kwargs: Any) -> Logger: # type: ignore[overr logger = ReplaySafeLoggerProvider(AttributedLoggerProvider()).get_logger("test") assert logger.custom == "custom-value" # type: ignore[attr-defined] - - -def _run_in_subprocess(code: str) -> None: - # Import-time behavior must be tested in a fresh interpreter so the - # simulated old opentelemetry-api is seen before temporalio imports it and - # no module state leaks into other tests. - result = subprocess.run( - [sys.executable, "-c", textwrap.dedent(code)], - capture_output=True, - text=True, - ) - assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" - - -def test_tracing_importable_without_logs_api(): - """opentelemetry-api < 1.15 has no opentelemetry._logs module at all; - tracing and metrics users must be unaffected and ReplaySafeLoggerProvider - must raise an actionable error on access. Blocking opentelemetry._logs - itself would also break the modern opentelemetry-sdk installed here, so - simulate by failing the guarded submodule import.""" - _run_in_subprocess( - """ - import sys - - sys.modules["temporalio.contrib.opentelemetry._logger_provider"] = None - - import temporalio.contrib.opentelemetry as otel_contrib - - assert otel_contrib.ReplaySafeTracerProvider is not None - assert otel_contrib.ReplaySafeMeterProvider is not None - assert otel_contrib.create_tracer_provider is not None - try: - otel_contrib.ReplaySafeLoggerProvider - except ImportError as err: - assert "opentelemetry-api >= 1.15" in str(err), str(err) - else: - raise AssertionError("expected ImportError accessing ReplaySafeLoggerProvider") - """ - ) diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py index 4c300ff31..cddd88b9a 100644 --- a/tests/contrib/opentelemetry/test_meter_provider.py +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -1,8 +1,5 @@ """Unit tests for ReplaySafeMeterProvider outside workflows.""" -import subprocess -import sys -import textwrap from collections.abc import Iterable from opentelemetry.context import Context @@ -225,71 +222,3 @@ def test_replay_safe_meter_provider_delegates_other_attributes(): provider = ReplaySafeMeterProvider(inner_provider) assert provider.force_flush() provider.shutdown() - - -def _run_in_subprocess(code: str) -> None: - # Import-time behavior must be tested in a fresh interpreter so the - # simulated old opentelemetry-api is seen before temporalio imports it and - # no module state leaks into other tests. - result = subprocess.run( - [sys.executable, "-c", textwrap.dedent(code)], - capture_output=True, - text=True, - ) - assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" - - -def test_module_imports_without_sync_gauge(): - """opentelemetry-api 1.12 through 1.22 has no opentelemetry.metrics._Gauge; - the module must still import and wrap the other instruments.""" - _run_in_subprocess( - """ - import opentelemetry.metrics - from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider - from opentelemetry.sdk.metrics.export import InMemoryMetricReader - - del opentelemetry.metrics._Gauge - - from temporalio.contrib.opentelemetry import ReplaySafeMeterProvider - - reader = InMemoryMetricReader() - provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) - provider.get_meter("test-meter").create_counter("counter").add(1) - - data = reader.get_metrics_data() - assert data is not None - metrics = [ - metric.name - for rm in data.resource_metrics - for sm in rm.scope_metrics - for metric in sm.metrics - ] - assert metrics == ["counter"], metrics - """ - ) - - -def test_tracing_importable_without_metrics_api(): - """opentelemetry-api < 1.12 has no opentelemetry.metrics module at all; - tracing users must be unaffected and ReplaySafeMeterProvider must raise an - actionable error on access. Blocking opentelemetry.metrics itself would - also break the modern opentelemetry-sdk trace module installed here, so - simulate by failing the guarded submodule import.""" - _run_in_subprocess( - """ - import sys - - sys.modules["temporalio.contrib.opentelemetry._meter_provider"] = None - - import temporalio.contrib.opentelemetry as otel_contrib - - assert otel_contrib.ReplaySafeTracerProvider is not None - assert otel_contrib.create_tracer_provider is not None - try: - otel_contrib.ReplaySafeMeterProvider - except ImportError as err: - assert "opentelemetry-api >= 1.12" in str(err), str(err) - else: - raise AssertionError("expected ImportError accessing ReplaySafeMeterProvider") - """ - ) diff --git a/uv.lock b/uv.lock index c825d5488..bcd32ae50 100644 --- a/uv.lock +++ b/uv.lock @@ -4716,10 +4716,10 @@ requires-dist = [ { name = "nexus-rpc", specifier = "==1.4.0" }, { name = "openai-agents", marker = "extra == 'openai-agents'", specifier = ">=0.17.5" }, { name = "opentelemetry-api", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, - { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.11.1,<2" }, + { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, { name = "opentelemetry-sdk", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, - { name = "opentelemetry-sdk", marker = "extra == 'opentelemetry'", specifier = ">=1.11.1,<2" }, + { name = "opentelemetry-sdk", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, { name = "opentelemetry-sdk-extension-aws", marker = "extra == 'lambda-worker-otel'", specifier = ">=2.0.0,<3" }, { name = "opentelemetry-semantic-conventions", marker = "extra == 'lambda-worker-otel'", specifier = ">=0.40b0,<1" }, { name = "protobuf", specifier = ">=3.20,<8.0.0" }, From f6ce144f2cf4e33217b26bb5084317adc5bccaeb Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 14:16:03 -0500 Subject: [PATCH 08/24] Call Logger base initializer in replay-safe logger wrapper Logger is an ABC whose __init__ records the instrumentation scope, so the wrapper now threads name/version/schema_url from get_logger through to the base class instead of relying on __getattr__ delegation for that state. Fixes the basedpyright reportMissingSuperCall error that failed lint. Also documents that opentelemetry._logs is the import path OpenTelemetry itself sanctions for the logs bridge API while it is pre-GA. --- .../contrib/opentelemetry/_logger_provider.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_logger_provider.py b/temporalio/contrib/opentelemetry/_logger_provider.py index e495f7480..2a9cac28f 100644 --- a/temporalio/contrib/opentelemetry/_logger_provider.py +++ b/temporalio/contrib/opentelemetry/_logger_provider.py @@ -2,11 +2,17 @@ from typing import TYPE_CHECKING, Any +# opentelemetry._logs is the import path OpenTelemetry itself documents for +# the logs bridge API while it is pre-GA (there is no non-underscore +# counterpart); the only alternative would be not gating ADK-style log +# emission at all. from opentelemetry._logs import Logger, LoggerProvider from temporalio import workflow if TYPE_CHECKING: + # _ExtendedAttributes is the annotation OpenTelemetry's own public + # get_logger signature uses; there is no public alias. from opentelemetry.util.types import _ExtendedAttributes @@ -17,7 +23,14 @@ def _skip_emitting() -> bool: class _ReplaySafeLogger(Logger): - def __init__(self, logger: Logger) -> None: + def __init__( + self, + logger: Logger, + name: str, + version: str | None = None, + schema_url: str | None = None, + ) -> None: + super().__init__(name, version=version, schema_url=schema_url) self._logger = logger def __getattr__(self, name: str) -> object: @@ -58,12 +71,11 @@ class ReplaySafeLoggerProvider(LoggerProvider): ReplaySafeLoggerProvider(my_logger_provider) ) - On opentelemetry-api 1.23+ proxy loggers late-bind, so calling - ``set_logger_provider`` after such libraries are imported still routes - their loggers through this wrapper; on 1.15 through 1.22 loggers obtained - before ``set_logger_provider`` stay no-op, so install this provider first. - ``set_logger_provider`` only takes effect once per process, so this - wrapper must be the one and only global logger provider ever set. + OpenTelemetry proxy loggers late-bind, so calling ``set_logger_provider`` + after such libraries are imported still routes their loggers through this + wrapper. However, ``set_logger_provider`` only takes effect once per + process, so this wrapper must be the one and only global logger provider + ever set. """ def __init__(self, logger_provider: LoggerProvider) -> None: @@ -107,4 +119,4 @@ def get_logger( inner = self._logger_provider.get_logger( name, version, schema_url, attributes ) - return _ReplaySafeLogger(inner) + return _ReplaySafeLogger(inner, name, version=version, schema_url=schema_url) From c9a19eb98e2e807c15b40b4a51b73748b9ab22de Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 14:16:14 -0500 Subject: [PATCH 09/24] Detect replay-unsafe OTel providers via public SDK types only The plugin now warns only when the global meter or tracer provider is an instance of the public OpenTelemetry SDK provider classes; everything else (unset proxies, no-ops, unknown wrappers) stays silent, which drops the private _ProxyMeterProvider/ProxyLoggerProvider imports. The SDK logger provider is not checked because its class is only importable from the underscore namespace opentelemetry.sdk._logs while OTel logs are pre-GA. The opentelemetry.sdk import is guarded for the SDK-less install case, the warning stacklevel walk uses inspect.currentframe, and the conftest OTel global resets are documented as the isolation pattern OpenTelemetry's own test suite uses. --- CHANGELOG.md | 2 +- .../contrib/google_adk_agents/README.md | 8 +- .../contrib/google_adk_agents/_plugin.py | 112 ++++-------------- tests/conftest.py | 12 +- .../google_adk_agents/test_replay_metrics.py | 74 +----------- 5 files changed, 42 insertions(+), 166 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02881fd1..05a8fb78e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ to include examples, links to docs, or any other relevant information. `temporalio.workflow.metric_meter()`. `temporalio.contrib.opentelemetry.ReplaySafeTracerProvider` is now also exported. `GoogleAdkPlugin` now warns at worker and replayer configuration time when the global - OpenTelemetry meter, tracer, or logger provider is positively identified as not replay-safe + OpenTelemetry meter or tracer provider is positively identified as not replay-safe (an OpenTelemetry SDK provider used directly). ### Changed diff --git a/temporalio/contrib/google_adk_agents/README.md b/temporalio/contrib/google_adk_agents/README.md index d2f540956..ec813ef03 100644 --- a/temporalio/contrib/google_adk_agents/README.md +++ b/temporalio/contrib/google_adk_agents/README.md @@ -247,9 +247,11 @@ logger_provider.add_log_record_processor(BatchLogRecordProcessor(my_log_exporter opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(logger_provider)) ``` -`GoogleAdkPlugin` warns at worker and replayer configuration time when a -global provider is installed that it can positively identify as not -replay-safe (an OpenTelemetry SDK provider used directly). +`GoogleAdkPlugin` warns at worker and replayer configuration time when the +global meter or tracer provider is positively identified as not replay-safe +(an OpenTelemetry SDK provider used directly). The global logger provider is +not checked because the OpenTelemetry logs SDK has no public import path yet, +but the same replay duplication applies to it. Recordings are first-execution-only, matching `temporalio.workflow.metric_meter()`: a retried workflow task re-executes diff --git a/temporalio/contrib/google_adk_agents/_plugin.py b/temporalio/contrib/google_adk_agents/_plugin.py index 2f1d34b71..f8cd0ed00 100644 --- a/temporalio/contrib/google_adk_agents/_plugin.py +++ b/temporalio/contrib/google_adk_agents/_plugin.py @@ -1,7 +1,7 @@ from __future__ import annotations import dataclasses -import sys +import inspect import time import uuid import warnings @@ -10,19 +10,8 @@ from types import FrameType from typing import Any -import opentelemetry._logs import opentelemetry.metrics import opentelemetry.trace -from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider -from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.sdk._logs import LoggerProvider as SdkLoggerProvider -from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider -from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider -from opentelemetry.trace import ( - NoOpTracerProvider, - ProxyTracerProvider, - TracerProvider, -) from temporalio import workflow from temporalio.contrib.google_adk_agents._mcp import TemporalMcpToolSetProvider @@ -30,11 +19,6 @@ invoke_model, invoke_model_streaming, ) -from temporalio.contrib.opentelemetry import ( - ReplaySafeLoggerProvider, - ReplaySafeMeterProvider, - ReplaySafeTracerProvider, -) from temporalio.contrib.pydantic import ( PydanticPayloadConverter, ToJsonOptions, @@ -48,61 +32,14 @@ ) from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner -# Each classifier below returns True when the provider is replay-safe -# (including providers that drop all recordings), False when positively -# identified as replay-unsafe (an OpenTelemetry SDK provider), and None when -# it cannot be classified. Unknown provider types (e.g. a custom provider -# delegating to a replay-safe one) must not warn: a false positive is worse -# than a missed warning. - - -def _meter_provider_replay_safe(provider: MeterProvider) -> bool | None: - if isinstance(provider, (ReplaySafeMeterProvider, NoOpMeterProvider)): - return True - try: - # Unlike tracing's public ProxyTracerProvider, the proxy (unset) meter - # provider has no public counterpart. Import the private class lazily - # so a moved or removed symbol cannot break module import; it is - # present in opentelemetry-api 1.12 through at least 1.42. - from opentelemetry.metrics._internal import _ProxyMeterProvider - - if isinstance(provider, _ProxyMeterProvider): - return True - except ImportError: - pass - return False if isinstance(provider, SdkMeterProvider) else None - - -def _tracer_provider_replay_safe(provider: TracerProvider) -> bool | None: - if isinstance( - provider, - (ReplaySafeTracerProvider, NoOpTracerProvider, ProxyTracerProvider), - ): - return True - return False if isinstance(provider, SdkTracerProvider) else None - - -def _logger_provider_replay_safe(provider: LoggerProvider) -> bool | None: - if isinstance(provider, (ReplaySafeLoggerProvider, NoOpLoggerProvider)): - return True - try: - # The proxy (unset) logger provider has no public counterpart either; - # present in opentelemetry-api 1.23 through at least 1.42. - from opentelemetry._logs._internal import ProxyLoggerProvider - - if isinstance(provider, ProxyLoggerProvider): - return True - except ImportError: - pass - return False if isinstance(provider, SdkLoggerProvider) else None - def _stacklevel_outside_temporalio() -> int: # Attribute provider warnings to the nearest frame outside temporalio, # e.g. the user's Worker(...)/Replayer(...) call or a user plugin that # delegates here, however many plugin frames sit in between. level = 1 - frame: FrameType | None = sys._getframe(1) + own_frame: FrameType | None = inspect.currentframe() + frame = own_frame.f_back if own_frame is not None else None while frame is not None: module = frame.f_globals.get("__name__", "") if module != "temporalio" and not module.startswith("temporalio."): @@ -116,10 +53,23 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: # ADK records metrics, spans, and log events through the process-global # OpenTelemetry providers from code that runs workflow-side, so a # non-replay-safe global provider re-emits that telemetry on every - # workflow replay. Unset (proxy) and no-op providers drop recordings and - # are fine. + # workflow replay. Warn only on providers positively identified as + # replay-unsafe: an OpenTelemetry SDK provider used directly as the + # global. Anything else stays silent -- unset (proxy) and no-op providers + # drop recordings, and unknown provider types (e.g. a custom provider + # delegating to a replay-safe one) cannot be classified, where a false + # positive is worse than a missed warning. The SDK logger provider is not + # checked because its class is only importable from the underscore + # namespace opentelemetry.sdk._logs while OpenTelemetry logs are pre-GA. + try: + from opentelemetry.sdk.metrics import MeterProvider as SdkMeterProvider + from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider + except ImportError: + # Without the opentelemetry-sdk package installed no SDK provider can + # exist, so there is nothing replay-unsafe to warn about. + return stacklevel = _stacklevel_outside_temporalio() - if _meter_provider_replay_safe(opentelemetry.metrics.get_meter_provider()) is False: + if isinstance(opentelemetry.metrics.get_meter_provider(), SdkMeterProvider): warnings.warn( "The global OpenTelemetry MeterProvider is not replay-safe: Google ADK " "records metrics from workflow code, so every workflow replay will " @@ -130,7 +80,7 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: UserWarning, stacklevel=stacklevel, ) - if _tracer_provider_replay_safe(opentelemetry.trace.get_tracer_provider()) is False: + if isinstance(opentelemetry.trace.get_tracer_provider(), SdkTracerProvider): warnings.warn( "The global OpenTelemetry TracerProvider is not replay-safe: Google ADK " "creates spans from workflow code, so every workflow replay will " @@ -140,17 +90,6 @@ def _warn_if_global_otel_providers_not_replay_safe() -> None: UserWarning, stacklevel=stacklevel, ) - if _logger_provider_replay_safe(opentelemetry._logs.get_logger_provider()) is False: - warnings.warn( - "The global OpenTelemetry LoggerProvider is not replay-safe: Google ADK " - "emits log events (e.g. gen_ai.choice) from workflow code, so every " - "workflow replay will re-emit them. Wrap your provider in " - "temporalio.contrib.opentelemetry.ReplaySafeLoggerProvider and make it " - "the first and only global provider set: " - "opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(provider))", - UserWarning, - stacklevel=stacklevel, - ) def setup_deterministic_runtime(): @@ -247,16 +186,17 @@ def workflow_runner(runner: WorkflowRunner | None) -> WorkflowRunner: ) def configure_worker(self, config: WorkerConfig) -> WorkerConfig: - """See base class. Also warns when the global OpenTelemetry providers - are not replay-safe, since ADK telemetry would duplicate on replay. + """See base class. Also warns when the global OpenTelemetry meter or + tracer provider is not replay-safe, since ADK telemetry would + duplicate on replay. """ _warn_if_global_otel_providers_not_replay_safe() return super().configure_worker(config) def configure_replayer(self, config: ReplayerConfig) -> ReplayerConfig: - """See base class. Also warns when the global OpenTelemetry providers - are not replay-safe, since every replayed workflow would re-emit ADK - telemetry. + """See base class. Also warns when the global OpenTelemetry meter or + tracer provider is not replay-safe, since every replayed workflow + would re-emit ADK telemetry. """ _warn_if_global_otel_providers_not_replay_safe() return super().configure_replayer(config) diff --git a/tests/conftest.py b/tests/conftest.py index ebdc52a67..533bcf0f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -262,9 +262,15 @@ def continue_as_new_suggest_history_count() -> int: return CONTINUE_AS_NEW_SUGGEST_HISTORY_COUNT +# OpenTelemetry's global providers are set-once per process with no public +# way to unset them, so tests needing their own provider must reset the +# globals directly -- the same isolation pattern OpenTelemetry's own test +# suite uses (opentelemetry.test.globals_test). + + @pytest.fixture def reset_otel_tracer_provider(): - """Reset global OpenTelemetry tracer provider state around tests.""" + """Isolate global OpenTelemetry tracer provider state around a test.""" opentelemetry.trace._TRACER_PROVIDER_SET_ONCE = Once() opentelemetry.trace._TRACER_PROVIDER = None yield @@ -274,7 +280,7 @@ def reset_otel_tracer_provider(): @pytest.fixture def reset_otel_meter_provider(): - """Reset global OpenTelemetry meter provider state around tests. + """Isolate global OpenTelemetry meter provider state around a test. Proxy meters/instruments already bound to a real provider stay bound after this reset; only the next set_meter_provider call rebinds them. Tests must @@ -290,7 +296,7 @@ def reset_otel_meter_provider(): @pytest.fixture def reset_otel_logger_provider(): - """Reset global OpenTelemetry logger provider state around tests. + """Isolate global OpenTelemetry logger provider state around a test. Unlike proxy meters, proxy loggers cache their real logger on first use and never rebind, even across a later set_logger_provider call. Tests diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index 6749e1671..a64546b93 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -11,7 +11,6 @@ """ import inspect -import sys import uuid import warnings from collections.abc import AsyncGenerator @@ -28,11 +27,8 @@ from google.adk.runners import InMemoryRunner from google.adk.utils.context_utils import Aclosing from google.genai import types -from opentelemetry._logs import NoOpLoggerProvider -from opentelemetry._logs._internal import ProxyLoggerProvider -from opentelemetry.metrics import Meter, NoOpMeterProvider +from opentelemetry.metrics import Meter from opentelemetry.metrics import MeterProvider as ApiMeterProvider -from opentelemetry.metrics._internal import _ProxyMeterProvider from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import ( InMemoryLogRecordExporter, @@ -41,7 +37,6 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.trace import TracerProvider as ApiTracerProvider from opentelemetry.trace import set_tracer_provider from opentelemetry.util.types import Attributes @@ -369,14 +364,6 @@ def test_plugin_warns_on_non_replay_safe_tracer_provider( GoogleAdkPlugin().configure_worker(_worker_config()) -def test_plugin_warns_on_non_replay_safe_logger_provider( - reset_otel_logger_provider, # type: ignore[reportUnusedParameter] -): - opentelemetry._logs.set_logger_provider(LoggerProvider()) - with pytest.warns(UserWarning, match="LoggerProvider is not replay-safe"): - GoogleAdkPlugin().configure_worker(_worker_config()) - - def test_plugin_warns_on_replayer_construction( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] ): @@ -428,65 +415,6 @@ def get_meter( return self._inner.get_meter(name, version, schema_url) -def test_meter_provider_replay_safety_classification(monkeypatch: pytest.MonkeyPatch): - from temporalio.contrib.google_adk_agents._plugin import ( - _meter_provider_replay_safe, - ) - - assert _meter_provider_replay_safe(ReplaySafeMeterProvider(MeterProvider())) is True - assert _meter_provider_replay_safe(NoOpMeterProvider()) is True - assert _meter_provider_replay_safe(_ProxyMeterProvider()) is True - assert _meter_provider_replay_safe(MeterProvider()) is False - # Unknown provider types (e.g. a custom provider delegating to a - # replay-safe one) are unclassifiable and must not warn. - assert _meter_provider_replay_safe(_DelegatingMeterProvider()) is None - - # SDK providers stay positively identified even when the private proxy - # class cannot be imported; the proxy instance becomes unclassifiable. - monkeypatch.setitem(sys.modules, "opentelemetry.metrics._internal", None) - assert _meter_provider_replay_safe(MeterProvider()) is False - assert _meter_provider_replay_safe(_ProxyMeterProvider()) is None - assert _meter_provider_replay_safe(NoOpMeterProvider()) is True - - -def test_tracer_provider_replay_safety_classification(): - from temporalio.contrib.google_adk_agents._plugin import ( - _tracer_provider_replay_safe, - ) - - class DelegatingTracerProvider(ApiTracerProvider): - def __init__(self) -> None: - self._inner = create_tracer_provider() - - def get_tracer(self, *args, **kwargs): - return self._inner.get_tracer(*args, **kwargs) - - assert _tracer_provider_replay_safe(create_tracer_provider()) is True - assert _tracer_provider_replay_safe(TracerProvider()) is False - assert _tracer_provider_replay_safe(DelegatingTracerProvider()) is None - - -def test_logger_provider_replay_safety_classification(): - from temporalio.contrib.google_adk_agents._plugin import ( - _logger_provider_replay_safe, - ) - - class DelegatingLoggerProvider(opentelemetry._logs.LoggerProvider): - def __init__(self) -> None: - self._inner = ReplaySafeLoggerProvider(LoggerProvider()) - - def get_logger(self, *args, **kwargs): - return self._inner.get_logger(*args, **kwargs) - - assert _logger_provider_replay_safe(ReplaySafeLoggerProvider(LoggerProvider())) is ( - True - ) - assert _logger_provider_replay_safe(NoOpLoggerProvider()) is True - assert _logger_provider_replay_safe(ProxyLoggerProvider()) is True - assert _logger_provider_replay_safe(LoggerProvider()) is False - assert _logger_provider_replay_safe(DelegatingLoggerProvider()) is None - - def test_plugin_does_not_warn_on_unknown_custom_provider( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] From c342030c688d77c61f35eda9567c12e4e26494d1 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 14:34:34 -0500 Subject: [PATCH 10/24] Align lambda-worker-otel extra floor with opentelemetry extra --- CHANGELOG.md | 9 +++++---- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05a8fb78e..19f1b519c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,10 +34,11 @@ to include examples, links to docs, or any other relevant information. ### Changed -- The `opentelemetry` extra now requires `opentelemetry-api`/`opentelemetry-sdk` `>= 1.24`, - aligning the declared floor with what `temporalio.contrib.opentelemetry` already required in - practice (it has depended on an API added in `opentelemetry-api` 1.24 since the tracing - integration was introduced). +- The `opentelemetry` and `lambda-worker-otel` extras now require + `opentelemetry-api`/`opentelemetry-sdk` `>= 1.24`, aligning the declared floor with what + `temporalio.contrib.opentelemetry` already required in practice (it has depended on an API + added in `opentelemetry-api` 1.24 since the tracing integration was introduced, and the + lambda worker builds on it). ### Deprecated diff --git a/pyproject.toml b/pyproject.toml index 05ffb1275..2c2c03436 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,8 +33,8 @@ google-adk = ["google-adk>=2.2.0,<3"] langgraph = ["langgraph>=1.1.0"] langsmith = ["langsmith>=0.7.34,<0.9"] lambda-worker-otel = [ - "opentelemetry-api>=1.11.1,<2", - "opentelemetry-sdk>=1.11.1,<2", + "opentelemetry-api>=1.24,<2", + "opentelemetry-sdk>=1.24,<2", "opentelemetry-exporter-otlp-proto-grpc>=1.11.1,<2", "opentelemetry-semantic-conventions>=0.40b0,<1", "opentelemetry-sdk-extension-aws>=2.0.0,<3", diff --git a/uv.lock b/uv.lock index bcd32ae50..9deca81af 100644 --- a/uv.lock +++ b/uv.lock @@ -4715,10 +4715,10 @@ requires-dist = [ { name = "mcp", marker = "extra == 'openai-agents'", specifier = ">=1.9.4,<2" }, { name = "nexus-rpc", specifier = "==1.4.0" }, { name = "openai-agents", marker = "extra == 'openai-agents'", specifier = ">=0.17.5" }, - { name = "opentelemetry-api", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, + { name = "opentelemetry-api", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.24,<2" }, { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, - { name = "opentelemetry-sdk", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, + { name = "opentelemetry-sdk", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.24,<2" }, { name = "opentelemetry-sdk", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, { name = "opentelemetry-sdk-extension-aws", marker = "extra == 'lambda-worker-otel'", specifier = ">=2.0.0,<3" }, { name = "opentelemetry-semantic-conventions", marker = "extra == 'lambda-worker-otel'", specifier = ">=0.40b0,<1" }, From a15d37d562a1db2dbfd649ba45db3944ce369d07 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 17:33:47 -0500 Subject: [PATCH 11/24] Skip exact-count telemetry tests on workflow task retry --- .../google_adk_agents/test_replay_metrics.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index a64546b93..2eb057a71 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -42,6 +42,7 @@ import temporalio.contrib.google_adk_agents.workflow from temporalio import activity, workflow +from temporalio.api.enums.v1 import EventType from temporalio.client import Client, WorkflowHistory from temporalio.contrib.google_adk_agents import GoogleAdkPlugin, TemporalModel from temporalio.contrib.opentelemetry import ( @@ -232,6 +233,10 @@ async def _run_once_and_replay( """Run the agent workflow once for real, then replay it num_replays times. Returns the number of real activity executions observed for this run. + Skips the calling test if the live run's history shows a workflow task + retry: a retried task legitimately re-records live telemetry + (at-least-once semantics), which would break the exact-count assertions + the callers make. """ LLMRegistry.register(ReplayMetricsModel) @@ -260,6 +265,19 @@ async def _run_once_and_replay( assert result == "warm and sunny" history = await handle.fetch_history() + if any( + event.event_type + in ( + EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED, + EventType.EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT, + ) + for event in history.events + ): + pytest.skip( + "Workflow task retried during the live run; exact telemetry " + "counts require a retry-free history" + ) + for _ in range(num_replays): await Replayer( workflows=[ReplayMetricsAgent], From f69559931ccc72f8776ddf7a4f5f171fcb834f0b Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 17:33:47 -0500 Subject: [PATCH 12/24] Fix ReplaySafeTracerProvider constructor docs and error typo --- temporalio/contrib/opentelemetry/_tracer_provider.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_tracer_provider.py b/temporalio/contrib/opentelemetry/_tracer_provider.py index 929f8bf27..24adeb780 100644 --- a/temporalio/contrib/opentelemetry/_tracer_provider.py +++ b/temporalio/contrib/opentelemetry/_tracer_provider.py @@ -174,16 +174,20 @@ def __init__( ): """Initialize the replay-safe tracer provider. + Prefer :py:func:`create_tracer_provider`, which constructs a provider + that satisfies these requirements. + Args: tracer_provider: The underlying OpenTelemetry TracerProvider to wrap. - Must use a _TemporalIdGenerator for replay safety. + Must use a ``TemporalIdGenerator`` for replay safety. + id_generator: The ``TemporalIdGenerator`` used by ``tracer_provider``. Raises: - ValueError: If the tracer provider doesn't use a _TemporalIdGenerator. + ValueError: If the tracer provider doesn't use a ``TemporalIdGenerator``. """ if not isinstance(tracer_provider.id_generator, TemporalIdGenerator): raise ValueError( - "ReplaySafeTracerProvider should only be used with a TemporalIdGenerator for replay safety. The given TracerProvider doesnt use one." + "ReplaySafeTracerProvider should only be used with a TemporalIdGenerator for replay safety. The given TracerProvider doesn't use one." ) self._id_generator = id_generator self._tracer_provider = tracer_provider From d06257ab88330b70f80d8c7753a73729cf801c75 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 17:33:47 -0500 Subject: [PATCH 13/24] Clarify replay-safe provider docs, changelog wording, and audit note --- CHANGELOG.md | 5 +++-- temporalio/contrib/google_adk_agents/_plugin.py | 4 ++++ temporalio/contrib/opentelemetry/__init__.py | 3 ++- temporalio/contrib/opentelemetry/_meter_provider.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f1b519c..d3ba12252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ to include examples, links to docs, or any other relevant information. ### Added - Added experimental `temporalio.contrib.opentelemetry.ReplaySafeMeterProvider` and - `ReplaySafeLoggerProvider`, wrapping OpenTelemetry providers that drop synchronous instrument - recordings and emitted log records made from workflow code during replay. Install them as the + `ReplaySafeLoggerProvider`, which wrap a user-supplied OpenTelemetry provider and drop + synchronous instrument recordings and emitted log records made from workflow code during + replay. Install them as the process-global providers when libraries record OpenTelemetry metrics or emit log events from workflow code (e.g. Google ADK) so that workflow replays (cache eviction, worker restarts, redeploys) do not duplicate telemetry; recordings are first-execution-only, matching diff --git a/temporalio/contrib/google_adk_agents/_plugin.py b/temporalio/contrib/google_adk_agents/_plugin.py index f8cd0ed00..9fc3c4d19 100644 --- a/temporalio/contrib/google_adk_agents/_plugin.py +++ b/temporalio/contrib/google_adk_agents/_plugin.py @@ -135,6 +135,10 @@ class GoogleAdkPlugin(SimplePlugin): This plugin configures: - Pydantic Payload Converter (required for ADK objects). - Sandbox Passthrough for google.adk and google.genai modules. + + At worker and replayer configuration time it also warns when the global + OpenTelemetry meter or tracer provider is not replay-safe, since ADK + telemetry recorded from workflow code would duplicate on replay. """ def __init__( diff --git a/temporalio/contrib/opentelemetry/__init__.py b/temporalio/contrib/opentelemetry/__init__.py index 159f7f8db..feafe91ae 100644 --- a/temporalio/contrib/opentelemetry/__init__.py +++ b/temporalio/contrib/opentelemetry/__init__.py @@ -2,7 +2,8 @@ This package provides OpenTelemetry tracing integration for Temporal workflows, activities, and other operations. It includes automatic span creation and -propagation for distributed tracing. +propagation for distributed tracing. It also provides replay-safe wrappers for +the global OpenTelemetry tracer, meter, and logger providers. """ from temporalio.contrib.opentelemetry._interceptor import ( diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index 2cfd1186a..164c63cfd 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -112,7 +112,7 @@ def set( class _ReplaySafeMeter(Meter): - # Overrides every Meter method as of opentelemetry-api 1.42. OTel adds new + # Overrides every Meter method as of opentelemetry-api 1.44. OTel adds new # instrument kinds as non-abstract no-op defaults on the Meter ABC (e.g. # create_gauge in 1.23), which __getattr__ cannot intercept, so new Meter # methods must be audited and overridden here on opentelemetry upgrades. From 961537b7990fccddcb20f3557da193c0a2e01db1 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 18:30:29 -0500 Subject: [PATCH 14/24] Construct the wrapped LoggerProvider in the log-events doc snippet --- temporalio/contrib/opentelemetry/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/temporalio/contrib/opentelemetry/README.md b/temporalio/contrib/opentelemetry/README.md index 0010cf73d..a81c40bbf 100644 --- a/temporalio/contrib/opentelemetry/README.md +++ b/temporalio/contrib/opentelemetry/README.md @@ -278,14 +278,15 @@ during replay: ```python import opentelemetry._logs from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from temporalio.contrib.opentelemetry import ReplaySafeLoggerProvider # set_logger_provider only takes effect once per process, so this wrapper # must be the first and only global logger provider set, installed before # any library emits log records. -opentelemetry._logs.set_logger_provider( - ReplaySafeLoggerProvider(my_logger_provider) -) +logger_provider = LoggerProvider() +logger_provider.add_log_record_processor(BatchLogRecordProcessor(my_log_exporter)) +opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(logger_provider)) ``` Emissions are first-execution-only: a retried workflow task re-executes live From 2c99db4de22cabaea301f8c5c8b0b8a78afe1d74 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 4 Aug 2026 18:30:29 -0500 Subject: [PATCH 15/24] Return only the execution count from _run_once_and_replay --- .../google_adk_agents/test_replay_metrics.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index 2eb057a71..96281eafc 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -43,7 +43,7 @@ import temporalio.contrib.google_adk_agents.workflow from temporalio import activity, workflow from temporalio.api.enums.v1 import EventType -from temporalio.client import Client, WorkflowHistory +from temporalio.client import Client from temporalio.contrib.google_adk_agents import GoogleAdkPlugin, TemporalModel from temporalio.contrib.opentelemetry import ( ReplaySafeLoggerProvider, @@ -227,9 +227,7 @@ def clear() -> None: clear() -async def _run_once_and_replay( - client: Client, num_replays: int -) -> tuple[int, WorkflowHistory]: +async def _run_once_and_replay(client: Client, num_replays: int) -> int: """Run the agent workflow once for real, then replay it num_replays times. Returns the number of real activity executions observed for this run. @@ -284,7 +282,7 @@ async def _run_once_and_replay( plugins=[GoogleAdkPlugin()], ).replay_workflow(history) - return activity_executions - activity_executions_before, history + return activity_executions - activity_executions_before async def test_replay_safe_meter_provider_suppresses_replay_metrics( @@ -296,7 +294,7 @@ async def test_replay_safe_meter_provider_suppresses_replay_metrics( ReplaySafeMeterProvider(MeterProvider(metric_readers=[reader])) ) - real_executions, _ = await _run_once_and_replay(client, num_replays=3) + real_executions = await _run_once_and_replay(client, num_replays=3) # First execution recorded exactly once (not suppressed), replays added # zero observations, and the activity never re-executed. @@ -313,7 +311,7 @@ async def test_replay_metrics_duplicate_without_replay_safe_meter_provider( reader = InMemoryMetricReader() opentelemetry.metrics.set_meter_provider(MeterProvider(metric_readers=[reader])) - real_executions, _ = await _run_once_and_replay(client, num_replays=3) + real_executions = await _run_once_and_replay(client, num_replays=3) assert real_executions == 1 assert adk_metric_counts(reader) == { @@ -336,7 +334,7 @@ async def test_replay_safe_logger_provider_suppresses_replay_log_events( provider, exporter = _in_memory_logger_provider() opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(provider)) - real_executions, _ = await _run_once_and_replay(client, num_replays=3) + real_executions = await _run_once_and_replay(client, num_replays=3) # First execution emitted exactly once (not suppressed), replays added # zero log events, and the activity never re-executed. @@ -354,7 +352,7 @@ async def test_replay_log_events_duplicate_without_replay_safe_logger_provider( provider, exporter = _in_memory_logger_provider() opentelemetry._logs.set_logger_provider(provider) - real_executions, _ = await _run_once_and_replay(client, num_replays=3) + real_executions = await _run_once_and_replay(client, num_replays=3) assert real_executions == 1 assert adk_log_event_counts(exporter) == { From 5cd180d126b0e51d2771f5bbc77584e4e6b3f9db Mon Sep 17 00:00:00 2001 From: DABH Date: Mon, 10 Aug 2026 22:07:57 -0500 Subject: [PATCH 16/24] Restore sdk-core submodule pointer from main --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index ce69d10f0..21fcc2952 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit ce69d10f0e80ec154264c3a7ed395af1e18aa796 +Subproject commit 21fcc2952257478489bd6e74cd0a65eb0b2c63be From 14205950f18de04e366d3fb18c7e0792818d564b Mon Sep 17 00:00:00 2001 From: DABH Date: Mon, 10 Aug 2026 22:31:18 -0500 Subject: [PATCH 17/24] Forward get_tracer attributes only when set --- .../contrib/opentelemetry/_tracer_provider.py | 22 +++++-- .../opentelemetry/test_tracer_provider.py | 61 +++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 tests/contrib/opentelemetry/test_tracer_provider.py diff --git a/temporalio/contrib/opentelemetry/_tracer_provider.py b/temporalio/contrib/opentelemetry/_tracer_provider.py index 24adeb780..683864e41 100644 --- a/temporalio/contrib/opentelemetry/_tracer_provider.py +++ b/temporalio/contrib/opentelemetry/_tracer_provider.py @@ -233,12 +233,22 @@ def get_tracer( Returns: A replay-safe tracer instance. """ - tracer = self._tracer_provider.get_tracer( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - attributes, - ) + # Forward attributes only when set: the parameter was added in + # opentelemetry 1.26 and passing it to older providers raises + # TypeError. + if attributes is None: + tracer = self._tracer_provider.get_tracer( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + ) + else: + tracer = self._tracer_provider.get_tracer( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + attributes, + ) return _ReplaySafeTracer(tracer) def id_generator(self) -> TemporalIdGenerator: diff --git a/tests/contrib/opentelemetry/test_tracer_provider.py b/tests/contrib/opentelemetry/test_tracer_provider.py new file mode 100644 index 000000000..1214407c3 --- /dev/null +++ b/tests/contrib/opentelemetry/test_tracer_provider.py @@ -0,0 +1,61 @@ +"""Tests for ReplaySafeTracerProvider.""" + +from __future__ import annotations + +from opentelemetry.trace import NoOpTracer, Tracer +from opentelemetry.util.types import Attributes + +from temporalio.contrib.opentelemetry import create_tracer_provider + + +def test_replay_safe_tracer_provider_supports_older_otel_signatures(): + """The get_tracer attributes parameter (opentelemetry 1.26) must only be + forwarded when set, so providers with the older three-parameter signature + (e.g. the OpenTelemetry SDK at the declared 1.24 floor) keep working.""" + provider = create_tracer_provider() + calls: list[tuple[str, str | None, str | None]] = [] + + def pre_126_get_tracer( + instrumenting_module_name: str, + instrumenting_library_version: str | None = None, + schema_url: str | None = None, + ) -> Tracer: + calls.append( + (instrumenting_module_name, instrumenting_library_version, schema_url) + ) + return NoOpTracer() + + setattr(provider._tracer_provider, "get_tracer", pre_126_get_tracer) + + provider.get_tracer("mod", "1.0", "https://schema") + + assert calls == [("mod", "1.0", "https://schema")] + + +def test_replay_safe_tracer_provider_forwards_attributes_when_set(): + """When the caller sets attributes, they are forwarded to the wrapped + provider (1.26+ signature).""" + provider = create_tracer_provider() + seen: list[tuple[str, str | None, str | None, Attributes | None]] = [] + + def recording_get_tracer( + instrumenting_module_name: str, + instrumenting_library_version: str | None = None, + schema_url: str | None = None, + attributes: Attributes | None = None, + ) -> Tracer: + seen.append( + ( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + attributes, + ) + ) + return NoOpTracer() + + setattr(provider._tracer_provider, "get_tracer", recording_get_tracer) + + provider.get_tracer("mod", attributes={"k": "v"}) + + assert seen == [("mod", None, None, {"k": "v"})] From 27a6e3eac589b29cec887d535fc6cf0668c59851 Mon Sep 17 00:00:00 2001 From: DABH Date: Mon, 10 Aug 2026 22:40:12 -0500 Subject: [PATCH 18/24] Raise opentelemetry floor to 1.26 and drop attributes conditionals The tracer provider has always forwarded the get_tracer attributes parameter (added in opentelemetry 1.26) unconditionally, so no release of this contrib ever worked for tracing below 1.26. Declaring 1.26 keeps the floor honest at the call level and removes the three conditional forwards for the attributes parameter; the version-gated forwards for parameters newer than the floor (context 1.28, histogram advisory 1.30, emit fields 1.38) remain. --- CHANGELOG.md | 9 ++--- pyproject.toml | 6 ++-- .../contrib/opentelemetry/_logger_provider.py | 10 +----- .../contrib/opentelemetry/_meter_provider.py | 10 +----- .../contrib/opentelemetry/_tracer_provider.py | 22 ++++-------- .../opentelemetry/test_logger_provider.py | 17 +++++----- .../opentelemetry/test_meter_provider.py | 20 ++++++----- .../opentelemetry/test_tracer_provider.py | 34 +++---------------- uv.lock | 8 ++--- 9 files changed, 45 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c8a0f8ac..d56cfe021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,10 +36,11 @@ to include examples, links to docs, or any other relevant information. ### Changed - The `opentelemetry` and `lambda-worker-otel` extras now require - `opentelemetry-api`/`opentelemetry-sdk` `>= 1.24`, aligning the declared floor with what - `temporalio.contrib.opentelemetry` already required in practice (it has depended on an API - added in `opentelemetry-api` 1.24 since the tracing integration was introduced, and the - lambda worker builds on it). + `opentelemetry-api`/`opentelemetry-sdk` `>= 1.26`, aligning the declared floor with what + `temporalio.contrib.opentelemetry` already required in practice: importing the contrib has + needed an API added in `opentelemetry-api` 1.24 since the tracing integration was + introduced, and `ReplaySafeTracerProvider.get_tracer` has always forwarded the + `attributes` parameter added in 1.26 (the lambda worker builds on the same contrib). - `temporalio.contrib.pydantic` converters now reuse Pydantic type adapters for repeated type hints instead of rebuilding their schemas for every payload, greatly speeding up decode of non-model hints such as discriminated diff --git a/pyproject.toml b/pyproject.toml index 075c09176..24f8d3494 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,15 +26,15 @@ classifiers = [ [project.optional-dependencies] grpc = ["grpcio>=1.48.2,<2"] -opentelemetry = ["opentelemetry-api>=1.24,<2", "opentelemetry-sdk>=1.24,<2"] +opentelemetry = ["opentelemetry-api>=1.26,<2", "opentelemetry-sdk>=1.26,<2"] pydantic = ["pydantic>=2.0.0,<3"] openai-agents = ["openai-agents>=0.17.5", "mcp>=1.9.4, <2"] google-adk = ["google-adk>=2.2.0,<3", "mcp>=1.24,<2"] langgraph = ["langgraph>=1.1.0"] langsmith = ["langsmith>=0.7.34,<0.9"] lambda-worker-otel = [ - "opentelemetry-api>=1.24,<2", - "opentelemetry-sdk>=1.24,<2", + "opentelemetry-api>=1.26,<2", + "opentelemetry-sdk>=1.26,<2", "opentelemetry-exporter-otlp-proto-grpc>=1.11.1,<2", "opentelemetry-semantic-conventions>=0.40b0,<1", "opentelemetry-sdk-extension-aws>=2.0.0,<3", diff --git a/temporalio/contrib/opentelemetry/_logger_provider.py b/temporalio/contrib/opentelemetry/_logger_provider.py index 2a9cac28f..80e878fdc 100644 --- a/temporalio/contrib/opentelemetry/_logger_provider.py +++ b/temporalio/contrib/opentelemetry/_logger_provider.py @@ -110,13 +110,5 @@ def get_logger( Returns: A replay-safe logger instance. """ - # Forward attributes only when set: the parameter was added in - # opentelemetry 1.26 and passing it to older providers raises - # TypeError. - if attributes is None: - inner = self._logger_provider.get_logger(name, version, schema_url) - else: - inner = self._logger_provider.get_logger( - name, version, schema_url, attributes - ) + inner = self._logger_provider.get_logger(name, version, schema_url, attributes) return _ReplaySafeLogger(inner, name, version=version, schema_url=schema_url) diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index 164c63cfd..c066d3784 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -266,13 +266,5 @@ def get_meter( Returns: A replay-safe meter instance. """ - # Forward attributes only when set: the parameter was added in - # opentelemetry 1.26 and passing it to older providers raises - # TypeError. - if attributes is None: - inner = self._meter_provider.get_meter(name, version, schema_url) - else: - inner = self._meter_provider.get_meter( - name, version, schema_url, attributes - ) + inner = self._meter_provider.get_meter(name, version, schema_url, attributes) return _ReplaySafeMeter(inner) diff --git a/temporalio/contrib/opentelemetry/_tracer_provider.py b/temporalio/contrib/opentelemetry/_tracer_provider.py index 683864e41..24adeb780 100644 --- a/temporalio/contrib/opentelemetry/_tracer_provider.py +++ b/temporalio/contrib/opentelemetry/_tracer_provider.py @@ -233,22 +233,12 @@ def get_tracer( Returns: A replay-safe tracer instance. """ - # Forward attributes only when set: the parameter was added in - # opentelemetry 1.26 and passing it to older providers raises - # TypeError. - if attributes is None: - tracer = self._tracer_provider.get_tracer( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - ) - else: - tracer = self._tracer_provider.get_tracer( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - attributes, - ) + tracer = self._tracer_provider.get_tracer( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + attributes, + ) return _ReplaySafeTracer(tracer) def id_generator(self) -> TemporalIdGenerator: diff --git a/tests/contrib/opentelemetry/test_logger_provider.py b/tests/contrib/opentelemetry/test_logger_provider.py index 94c2a5d0a..8508f0a2d 100644 --- a/tests/contrib/opentelemetry/test_logger_provider.py +++ b/tests/contrib/opentelemetry/test_logger_provider.py @@ -72,9 +72,9 @@ def get_logger( def test_replay_safe_logger_provider_supports_older_otel_signatures(): - """Newer opentelemetry-api parameters (get_logger attributes, 1.26; emit - keyword fields, 1.38) must only be forwarded when the caller passes them, - so loggers and providers with older signatures keep working.""" + """Newer opentelemetry-api parameters (emit keyword fields, 1.38) must + only be forwarded when the caller passes them, so loggers with older + signatures keep working.""" class Pre138Logger(NoOpLogger): def __init__(self) -> None: @@ -84,26 +84,27 @@ def __init__(self) -> None: def emit(self, record: LogRecord) -> None: # type: ignore[override] self.records.append(record) - class Pre126LoggerProvider(NoOpLoggerProvider): + class Pre138LoggerProvider(NoOpLoggerProvider): def __init__(self) -> None: self.logger = Pre138Logger() - self.get_logger_calls: list[tuple[str, str | None, str | None]] = [] + self.get_logger_calls: list[tuple[str, str | None, str | None, object]] = [] def get_logger( # type: ignore[override] self, name: str, version: str | None = None, schema_url: str | None = None, + attributes: object = None, ) -> Logger: - self.get_logger_calls.append((name, version, schema_url)) + self.get_logger_calls.append((name, version, schema_url, attributes)) return self.logger - inner_provider = Pre126LoggerProvider() + inner_provider = Pre138LoggerProvider() provider = ReplaySafeLoggerProvider(inner_provider) record = LogRecord(event_name="event", body="hello") provider.get_logger("test-logger").emit(record) - assert inner_provider.get_logger_calls == [("test-logger", None, None)] + assert inner_provider.get_logger_calls == [("test-logger", None, None, None)] assert inner_provider.logger.records == [record] diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py index cddd88b9a..9b4fb8380 100644 --- a/tests/contrib/opentelemetry/test_meter_provider.py +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -104,10 +104,9 @@ def get_meter( def test_replay_safe_meter_provider_supports_older_otel_signatures(): - """Newer opentelemetry-api parameters (get_meter attributes, 1.26; sync - instrument context, 1.28; create_histogram - explicit_bucket_boundaries_advisory, 1.30) must only be forwarded when - set, so providers with older signatures keep working.""" + """Newer opentelemetry-api parameters (sync instrument context, 1.28; + create_histogram explicit_bucket_boundaries_advisory, 1.30) must only be + forwarded when set, so providers with older signatures keep working.""" class Pre128Counter(Counter): def __init__(self) -> None: @@ -143,27 +142,30 @@ def create_histogram( # type: ignore[override] self.histogram_calls.append((name, unit, description)) return super().create_histogram(name, unit, description) - class Pre126MeterProvider(NoOpMeterProvider): + class Pre128MeterProvider(NoOpMeterProvider): def __init__(self) -> None: self.meter = Pre130Meter() - self.get_meter_calls: list[tuple[str, str | None, str | None]] = [] + self.get_meter_calls: list[ + tuple[str, str | None, str | None, Attributes | None] + ] = [] def get_meter( # type: ignore[override] self, name: str, version: str | None = None, schema_url: str | None = None, + attributes: Attributes | None = None, ) -> Meter: - self.get_meter_calls.append((name, version, schema_url)) + self.get_meter_calls.append((name, version, schema_url, attributes)) return self.meter - inner_provider = Pre126MeterProvider() + inner_provider = Pre128MeterProvider() provider = ReplaySafeMeterProvider(inner_provider) meter = provider.get_meter("test-meter") meter.create_histogram("histogram").record(1) meter.create_counter("counter").add(2, {"attr": "val"}) - assert inner_provider.get_meter_calls == [("test-meter", None, None)] + assert inner_provider.get_meter_calls == [("test-meter", None, None, None)] assert inner_provider.meter.histogram_calls == [("histogram", "", "")] assert inner_provider.meter.counter.calls == [(2, {"attr": "val"})] diff --git a/tests/contrib/opentelemetry/test_tracer_provider.py b/tests/contrib/opentelemetry/test_tracer_provider.py index 1214407c3..165fa62a4 100644 --- a/tests/contrib/opentelemetry/test_tracer_provider.py +++ b/tests/contrib/opentelemetry/test_tracer_provider.py @@ -8,33 +8,9 @@ from temporalio.contrib.opentelemetry import create_tracer_provider -def test_replay_safe_tracer_provider_supports_older_otel_signatures(): - """The get_tracer attributes parameter (opentelemetry 1.26) must only be - forwarded when set, so providers with the older three-parameter signature - (e.g. the OpenTelemetry SDK at the declared 1.24 floor) keep working.""" - provider = create_tracer_provider() - calls: list[tuple[str, str | None, str | None]] = [] - - def pre_126_get_tracer( - instrumenting_module_name: str, - instrumenting_library_version: str | None = None, - schema_url: str | None = None, - ) -> Tracer: - calls.append( - (instrumenting_module_name, instrumenting_library_version, schema_url) - ) - return NoOpTracer() - - setattr(provider._tracer_provider, "get_tracer", pre_126_get_tracer) - - provider.get_tracer("mod", "1.0", "https://schema") - - assert calls == [("mod", "1.0", "https://schema")] - - -def test_replay_safe_tracer_provider_forwards_attributes_when_set(): - """When the caller sets attributes, they are forwarded to the wrapped - provider (1.26+ signature).""" +def test_replay_safe_tracer_provider_delegates_get_tracer_arguments(): + """get_tracer forwards all arguments, including attributes (1.26+), to + the wrapped provider.""" provider = create_tracer_provider() seen: list[tuple[str, str | None, str | None, Attributes | None]] = [] @@ -56,6 +32,6 @@ def recording_get_tracer( setattr(provider._tracer_provider, "get_tracer", recording_get_tracer) - provider.get_tracer("mod", attributes={"k": "v"}) + provider.get_tracer("mod", "1.0", "https://schema", {"k": "v"}) - assert seen == [("mod", None, None, {"k": "v"})] + assert seen == [("mod", "1.0", "https://schema", {"k": "v"})] diff --git a/uv.lock b/uv.lock index a1ca03312..6ab61156b 100644 --- a/uv.lock +++ b/uv.lock @@ -4717,11 +4717,11 @@ requires-dist = [ { name = "mcp", marker = "extra == 'openai-agents'", specifier = ">=1.9.4,<2" }, { name = "nexus-rpc", specifier = "==1.4.0" }, { name = "openai-agents", marker = "extra == 'openai-agents'", specifier = ">=0.17.5" }, - { name = "opentelemetry-api", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.24,<2" }, - { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, + { name = "opentelemetry-api", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.26,<2" }, + { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.26,<2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.11.1,<2" }, - { name = "opentelemetry-sdk", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.24,<2" }, - { name = "opentelemetry-sdk", marker = "extra == 'opentelemetry'", specifier = ">=1.24,<2" }, + { name = "opentelemetry-sdk", marker = "extra == 'lambda-worker-otel'", specifier = ">=1.26,<2" }, + { name = "opentelemetry-sdk", marker = "extra == 'opentelemetry'", specifier = ">=1.26,<2" }, { name = "opentelemetry-sdk-extension-aws", marker = "extra == 'lambda-worker-otel'", specifier = ">=2.0.0,<3" }, { name = "opentelemetry-semantic-conventions", marker = "extra == 'lambda-worker-otel'", specifier = ">=0.40b0,<1" }, { name = "protobuf", specifier = ">=3.20,<8.0.0" }, From 334a076a1014691af6d77395b18650c1164fea69 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 14:04:58 -0500 Subject: [PATCH 19/24] Condense changelog entries --- CHANGELOG.md | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d56cfe021..debcddc41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,26 +21,16 @@ to include examples, links to docs, or any other relevant information. ### Added - Added experimental `temporalio.contrib.opentelemetry.ReplaySafeMeterProvider` and - `ReplaySafeLoggerProvider`, which wrap a user-supplied OpenTelemetry provider and drop - synchronous instrument recordings and emitted log records made from workflow code during - replay. Install them as the - process-global providers when libraries record OpenTelemetry metrics or emit log events from - workflow code (e.g. Google ADK) so that workflow replays (cache eviction, worker restarts, - redeploys) do not duplicate telemetry; recordings are first-execution-only, matching - `temporalio.workflow.metric_meter()`. - `temporalio.contrib.opentelemetry.ReplaySafeTracerProvider` is now also exported. - `GoogleAdkPlugin` now warns at worker and replayer configuration time when the global - OpenTelemetry meter or tracer provider is positively identified as not replay-safe - (an OpenTelemetry SDK provider used directly). + `ReplaySafeLoggerProvider` (and exported `ReplaySafeTracerProvider`): wrap an + OpenTelemetry provider so metrics and log events recorded from workflow code (e.g. by + Google ADK) are not duplicated on replay. `GoogleAdkPlugin` warns when a global OTel + provider is not replay-safe. ### Changed - The `opentelemetry` and `lambda-worker-otel` extras now require - `opentelemetry-api`/`opentelemetry-sdk` `>= 1.26`, aligning the declared floor with what - `temporalio.contrib.opentelemetry` already required in practice: importing the contrib has - needed an API added in `opentelemetry-api` 1.24 since the tracing integration was - introduced, and `ReplaySafeTracerProvider.get_tracer` has always forwarded the - `attributes` parameter added in 1.26 (the lambda worker builds on the same contrib). + `opentelemetry-api`/`opentelemetry-sdk` `>= 1.26`, matching what + `temporalio.contrib.opentelemetry` already required in practice. - `temporalio.contrib.pydantic` converters now reuse Pydantic type adapters for repeated type hints instead of rebuilding their schemas for every payload, greatly speeding up decode of non-model hints such as discriminated From 4278a3094185601182fd0c4e3b06eafd08074d56 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 16:00:32 -0500 Subject: [PATCH 20/24] Gate replay-safe telemetry on history-event replay Queries and update validators are live, at-most-once-per-request operations even when they execute while the workflow is replaying, so ReplaySafeMeterProvider and ReplaySafeLoggerProvider must not drop their recordings. Gate on is_replaying_history_events() instead of is_replaying(), matching the replay-safe tracer. --- temporalio/contrib/opentelemetry/README.md | 14 +- .../contrib/opentelemetry/_logger_provider.py | 22 ++- .../contrib/opentelemetry/_meter_provider.py | 25 +-- .../opentelemetry/test_logger_provider.py | 48 +++++- .../opentelemetry/test_meter_provider.py | 151 +++++++++++++++++- 5 files changed, 233 insertions(+), 27 deletions(-) diff --git a/temporalio/contrib/opentelemetry/README.md b/temporalio/contrib/opentelemetry/README.md index a81c40bbf..9b0ed79da 100644 --- a/temporalio/contrib/opentelemetry/README.md +++ b/temporalio/contrib/opentelemetry/README.md @@ -246,8 +246,8 @@ eviction, worker restart, redeploy), so a plain global meter provider re-records those metrics on each replay, inflating counts. `ReplaySafeMeterProvider` wraps your meter provider so synchronous instrument -recordings made from workflow code are dropped during replay, mirroring what -`create_tracer_provider()` does for spans: +recordings made from workflow code are dropped while the workflow is replaying +history events, mirroring what `create_tracer_provider()` does for spans: ```python import opentelemetry.metrics @@ -263,7 +263,9 @@ opentelemetry.metrics.set_meter_provider( ``` Recordings are first-execution-only, matching `workflow.metric_meter()`: a -retried workflow task re-executes live and can record again. Observable +retried workflow task re-executes live and can record again. Queries and +update validators are live, once-per-request operations even when they run +while the workflow is replaying, so their recordings are kept. Observable (asynchronous) instruments and recordings made outside workflows pass through untouched. @@ -273,7 +275,7 @@ Libraries may also emit OpenTelemetry log records through the process-global logger provider from workflow code (e.g. Google ADK's `gen_ai.*` events), which duplicate on every replay the same way. `ReplaySafeLoggerProvider` wraps your logger provider so records emitted from workflow code are dropped -during replay: +while the workflow is replaying history events: ```python import opentelemetry._logs @@ -290,7 +292,9 @@ opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(logger_provider ``` Emissions are first-execution-only: a retried workflow task re-executes live -and can emit again. Emissions outside workflows pass through untouched. +and can emit again. Queries and update validators are live, once-per-request +operations even when they run while the workflow is replaying, so their +emissions are kept. Emissions outside workflows pass through untouched. ## Best Practices diff --git a/temporalio/contrib/opentelemetry/_logger_provider.py b/temporalio/contrib/opentelemetry/_logger_provider.py index 80e878fdc..71fdd9f5f 100644 --- a/temporalio/contrib/opentelemetry/_logger_provider.py +++ b/temporalio/contrib/opentelemetry/_logger_provider.py @@ -17,9 +17,12 @@ def _skip_emitting() -> bool: - # in_workflow() must be evaluated first: is_replaying() requires an active - # workflow context. - return workflow.in_workflow() and workflow.unsafe.is_replaying() + # in_workflow() must be evaluated first: is_replaying_history_events() + # requires an active workflow context. The history-events predicate is + # deliberate (not is_replaying()): queries and update validators are live, + # at-most-once-per-request operations even when they execute while the + # workflow is replaying, so their emissions must not be dropped. + return workflow.in_workflow() and workflow.unsafe.is_replaying_history_events() class _ReplaySafeLogger(Logger): @@ -56,13 +59,16 @@ class ReplaySafeLoggerProvider(LoggerProvider): This logger provider wraps an OpenTelemetry LoggerProvider and drops log records emitted (``Logger.emit()``) from workflow code while the workflow - is replaying. Without this, libraries that emit OpenTelemetry log records - from workflow code (e.g. ``google-adk``'s ``gen_ai.*`` events) re-emit - every record on each replay, duplicating telemetry. + is replaying history events. Without this, libraries that emit + OpenTelemetry log records from workflow code (e.g. ``google-adk``'s + ``gen_ai.*`` events) re-emit every record on each replay, duplicating + telemetry. Emissions are therefore first-execution-only: a workflow task retry - re-executes live and can emit again. Emissions outside workflows are - unaffected. + re-executes live and can emit again. Queries and update validators are + live, at-most-once-per-request operations even when they execute while the + workflow is replaying, so their emissions are kept. Emissions outside + workflows are unaffected. Install this as the process-global logger provider before any library (e.g. ``google-adk``) obtains loggers:: diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index c066d3784..a0eea664a 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -30,9 +30,12 @@ def _forward_context_kwarg(context: Context | None) -> dict[str, Any]: def _skip_recording() -> bool: - # in_workflow() must be evaluated first: is_replaying() requires an active - # workflow context. - return workflow.in_workflow() and workflow.unsafe.is_replaying() + # in_workflow() must be evaluated first: is_replaying_history_events() + # requires an active workflow context. The history-events predicate is + # deliberate (not is_replaying()): queries and update validators are live, + # at-most-once-per-request operations even when they execute while the + # workflow is replaying, so their recordings must not be dropped. + return workflow.in_workflow() and workflow.unsafe.is_replaying_history_events() class _ReplaySafeCounter(Counter): @@ -209,16 +212,18 @@ class ReplaySafeMeterProvider(MeterProvider): This meter provider wraps an OpenTelemetry MeterProvider and drops synchronous instrument recordings (counter ``add()``, up-down counter ``add()``, histogram ``record()``, and gauge ``set()``) made from workflow - code while the workflow is replaying. Without this, libraries that record - metrics from workflow code (e.g. ``google-adk``) re-record every - measurement on each replay, inflating counts. + code while the workflow is replaying history events. Without this, + libraries that record metrics from workflow code (e.g. ``google-adk``) + re-record every measurement on each replay, inflating counts. Recordings are therefore first-execution-only, matching :py:meth:`temporalio.workflow.metric_meter`: a workflow task retry - re-executes live and can record again. Observable (asynchronous) - instruments pass through untouched since their callbacks run on the metric - reader's collect thread, never inside workflow code. Recordings outside - workflows are unaffected. + re-executes live and can record again. Queries and update validators are + live, at-most-once-per-request operations even when they execute while the + workflow is replaying, so their recordings are kept. Observable + (asynchronous) instruments pass through untouched since their callbacks + run on the metric reader's collect thread, never inside workflow code. + Recordings outside workflows are unaffected. Install this as the process-global meter provider before any library (e.g. ``google-adk``) creates instruments:: diff --git a/tests/contrib/opentelemetry/test_logger_provider.py b/tests/contrib/opentelemetry/test_logger_provider.py index 8508f0a2d..f26f8b508 100644 --- a/tests/contrib/opentelemetry/test_logger_provider.py +++ b/tests/contrib/opentelemetry/test_logger_provider.py @@ -1,6 +1,9 @@ -"""Unit tests for ReplaySafeLoggerProvider outside workflows.""" +"""Unit tests for ReplaySafeLoggerProvider.""" +from collections.abc import Iterator +from contextlib import contextmanager from typing import Any +from unittest.mock import patch from opentelemetry._logs import ( Logger, @@ -16,6 +19,7 @@ ) from opentelemetry.util.types import _ExtendedAttributes +from temporalio import workflow from temporalio.contrib.opentelemetry import ReplaySafeLoggerProvider @@ -108,6 +112,48 @@ def get_logger( # type: ignore[override] assert inner_provider.logger.records == [record] +@contextmanager +def _workflow_replay_state(*, replaying_history_events: bool) -> Iterator[None]: + """Simulate workflow context during replay. When replaying_history_events + is False this is the query/update-validator state: is_replaying() is True + but is_replaying_history_events() is False.""" + with ( + patch.object(workflow, "in_workflow", return_value=True), + patch.object(workflow.unsafe, "is_replaying", return_value=True), + patch.object( + workflow.unsafe, + "is_replaying_history_events", + return_value=replaying_history_events, + ), + ): + yield + + +def test_replay_safe_logger_provider_drops_emissions_replaying_history_events(): + provider, exporter = _sdk_provider() + logger = provider.get_logger("test-logger") + + with _workflow_replay_state(replaying_history_events=True): + logger.emit(LogRecord(event_name="replayed", body="dropped")) + + assert not exporter.get_finished_logs() + + +def test_replay_safe_logger_provider_emits_from_live_operations_during_replay(): + """Queries and update validators execute at most once per request even + when the workflow is replaying, so the gate must use + is_replaying_history_events(), not is_replaying(), and keep their + emissions.""" + provider, exporter = _sdk_provider() + logger = provider.get_logger("test-logger") + + with _workflow_replay_state(replaying_history_events=False): + logger.emit(LogRecord(event_name="live-query", body="kept")) + + records = [log.log_record for log in exporter.get_finished_logs()] + assert [(r.event_name, r.body) for r in records] == [("live-query", "kept")] + + def test_replay_safe_logger_provider_delegates_other_attributes(): provider, _ = _sdk_provider() assert provider.force_flush() diff --git a/tests/contrib/opentelemetry/test_meter_provider.py b/tests/contrib/opentelemetry/test_meter_provider.py index 9b4fb8380..3cefbc6e2 100644 --- a/tests/contrib/opentelemetry/test_meter_provider.py +++ b/tests/contrib/opentelemetry/test_meter_provider.py @@ -1,7 +1,12 @@ -"""Unit tests for ReplaySafeMeterProvider outside workflows.""" +"""Tests for ReplaySafeMeterProvider.""" -from collections.abc import Iterable +import uuid +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from unittest.mock import patch +import opentelemetry.metrics +import pytest from opentelemetry.context import Context from opentelemetry.metrics import ( CallbackOptions, @@ -17,13 +22,19 @@ from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.util.types import Attributes +from temporalio import workflow +from temporalio.api.enums.v1 import EventType +from temporalio.client import Client from temporalio.contrib.opentelemetry import ReplaySafeMeterProvider +from temporalio.worker import UnsandboxedWorkflowRunner +from tests.helpers import assert_eventually, new_worker def _metric_data_points(reader: InMemoryMetricReader) -> dict[str, list]: points: dict[str, list] = {} data = reader.get_metrics_data() - assert data is not None + if data is None: + return points for rm in data.resource_metrics: for sm in rm.scope_metrics: for metric in sm.metrics: @@ -224,3 +235,137 @@ def test_replay_safe_meter_provider_delegates_other_attributes(): provider = ReplaySafeMeterProvider(inner_provider) assert provider.force_flush() provider.shutdown() + + +@contextmanager +def _workflow_replay_state(*, replaying_history_events: bool) -> Iterator[None]: + """Simulate workflow context during replay. When replaying_history_events + is False this is the query/update-validator state: is_replaying() is True + but is_replaying_history_events() is False.""" + with ( + patch.object(workflow, "in_workflow", return_value=True), + patch.object(workflow.unsafe, "is_replaying", return_value=True), + patch.object( + workflow.unsafe, + "is_replaying_history_events", + return_value=replaying_history_events, + ), + ): + yield + + +def test_replay_safe_meter_provider_drops_recordings_replaying_history_events(): + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + counter = provider.get_meter("test-meter").create_counter("counter") + + with _workflow_replay_state(replaying_history_events=True): + counter.add(1) + + assert not _metric_data_points(reader).get("counter") + + +def test_replay_safe_meter_provider_records_from_live_operations_during_replay(): + """Queries and update validators execute at most once per request even + when the workflow is replaying, so the gate must use + is_replaying_history_events(), not is_replaying(), and keep their + recordings.""" + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + counter = provider.get_meter("test-meter").create_counter("counter") + + with _workflow_replay_state(replaying_history_events=False): + counter.add(1) + + assert _metric_data_points(reader)["counter"][0].value == 1 + + +@workflow.defn +class QueryDuringReplayWorkflow: + def __init__(self) -> None: + self._proceed = False + + @workflow.run + async def run(self) -> None: + opentelemetry.metrics.get_meter("replay-query-meter").create_counter( + "run_counter" + ).add(1) + await workflow.wait_condition(lambda: self._proceed) + + @workflow.signal + def proceed(self) -> None: + self._proceed = True + + @workflow.query + def query_and_record(self) -> bool: + opentelemetry.metrics.get_meter("replay-query-meter").create_counter( + "query_counter" + ).add(1) + return workflow.unsafe.is_replaying() + + +async def test_replay_safe_meter_provider_records_query_during_replay( + client: Client, + reset_otel_meter_provider, # type: ignore[reportUnusedParameter] +): + """End-to-end check of the replay predicate: with the workflow cache + disabled, a query forces a full history replay before its handler runs. + The replayed run() recording must be dropped while the query handler's + recording -- live, once per request -- must be kept.""" + reader = InMemoryMetricReader() + provider = ReplaySafeMeterProvider(SdkMeterProvider(metric_readers=[reader])) + opentelemetry.metrics.set_meter_provider(provider) + assert opentelemetry.metrics.get_meter_provider() is provider + + async with new_worker( + client, + QueryDuringReplayWorkflow, + # No cache, so the query task must replay history from scratch. + max_cached_workflows=0, + # Unsandboxed, so workflow code sees this process's global provider. + workflow_runner=UnsandboxedWorkflowRunner(), + ) as worker: + handle = await client.start_workflow( + QueryDuringReplayWorkflow.run, + id=f"query-during-replay-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + # Query only after the first workflow task completes: the live run() + # recording then exists and the evicted workflow must replay it. + async def first_workflow_task_completed() -> None: + history = await handle.fetch_history() + assert any( + event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED + for event in history.events + ) + + await assert_eventually(first_workflow_task_completed) + + was_replaying = await handle.query(QueryDuringReplayWorkflow.query_and_record) + # Pins that the disputed condition was exercised: the handler ran with + # is_replaying() True, and its recording was still kept below. + assert was_replaying + + await handle.signal(QueryDuringReplayWorkflow.proceed) + await handle.result() + history = await handle.fetch_history() + + if any( + event.event_type + in ( + EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED, + EventType.EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT, + ) + for event in history.events + ): + pytest.skip( + "Workflow task retried during the live run; exact telemetry " + "counts require a retry-free history" + ) + + points = _metric_data_points(reader) + # Live run recorded once; the query task's replay of run() was dropped. + assert points["run_counter"][0].value == 1 + # The query handler's recording during that replay was kept. + assert points["query_counter"][0].value == 1 From c0c2dc8e9141e2c3c9d17ccf0f355000500ba9d1 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 16:00:58 -0500 Subject: [PATCH 21/24] Clarify logs bridge import path comment The Logs API specification is stable; what keeps the import private is that opentelemetry-python has not promoted opentelemetry._logs to a public namespace. --- temporalio/contrib/opentelemetry/_logger_provider.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_logger_provider.py b/temporalio/contrib/opentelemetry/_logger_provider.py index 71fdd9f5f..e197dbd3f 100644 --- a/temporalio/contrib/opentelemetry/_logger_provider.py +++ b/temporalio/contrib/opentelemetry/_logger_provider.py @@ -2,10 +2,11 @@ from typing import TYPE_CHECKING, Any -# opentelemetry._logs is the import path OpenTelemetry itself documents for -# the logs bridge API while it is pre-GA (there is no non-underscore -# counterpart); the only alternative would be not gating ADK-style log -# emission at all. +# The OpenTelemetry Logs API specification is stable, but opentelemetry-python +# exposes the logs bridge API only under the private opentelemetry._logs +# module path (there is no public opentelemetry.logs as of 1.42), so _logs is +# the import path OpenTelemetry itself documents until the Python SIG promotes +# it to a public namespace. from opentelemetry._logs import Logger, LoggerProvider from temporalio import workflow From 1a5b1061b85a684840a79a2c578c3b63618080f3 Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 16:00:58 -0500 Subject: [PATCH 22/24] Delegate add_link in replay-safe spans The Span ABC ships add_link as a non-abstract warn-and-no-op default that __getattr__ cannot intercept, so links added through the wrapper after span creation were silently dropped. --- CHANGELOG.md | 4 ++++ temporalio/contrib/opentelemetry/_tracer_provider.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 458485eab..8b32400bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,10 @@ to include examples, links to docs, or any other relevant information. ### Fixed +- `temporalio.contrib.opentelemetry` replay-safe spans now delegate + `Span.add_link` to the wrapped span. Previously the wrapper inherited + OpenTelemetry's non-abstract no-op default, silently dropping links added + after span creation. - The `google-adk` extra now depends on `mcp`, so fresh installs of `temporalio[google-adk]` can import `temporalio.contrib.google_adk_agents` without separately installing `mcp`. Previously the import failed with an diff --git a/temporalio/contrib/opentelemetry/_tracer_provider.py b/temporalio/contrib/opentelemetry/_tracer_provider.py index 24adeb780..f2ad336d5 100644 --- a/temporalio/contrib/opentelemetry/_tracer_provider.py +++ b/temporalio/contrib/opentelemetry/_tracer_provider.py @@ -69,6 +69,16 @@ def add_event( ) -> None: self._span.add_event(name, attributes, timestamp) + def add_link( + self, + context: SpanContext, + attributes: types.Attributes = None, + ) -> None: + # Must be overridden explicitly: the Span ABC ships add_link as a + # non-abstract warn-and-no-op default that __getattr__ cannot + # intercept, which would silently drop links added after creation. + self._span.add_link(context, attributes) + def update_name(self, name: str) -> None: self._span.update_name(name) From f3c03ba6334c39ada3082ac6a842b5d4dafaf35a Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 16:01:24 -0500 Subject: [PATCH 23/24] Add drift-guard tests for OTel wrapper ABCs OTel adds API surface as non-abstract no-op defaults, so a wrapper method left un-overridden silently swallows telemetry instead of delegating. Enumerate each wrapped ABC's public and abstract surface by reflection so new opentelemetry-api surface fails loudly. --- .../contrib/opentelemetry/_meter_provider.py | 2 + .../opentelemetry/test_wrapper_abc_drift.py | 111 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 tests/contrib/opentelemetry/test_wrapper_abc_drift.py diff --git a/temporalio/contrib/opentelemetry/_meter_provider.py b/temporalio/contrib/opentelemetry/_meter_provider.py index a0eea664a..6d49ccc33 100644 --- a/temporalio/contrib/opentelemetry/_meter_provider.py +++ b/temporalio/contrib/opentelemetry/_meter_provider.py @@ -119,6 +119,8 @@ class _ReplaySafeMeter(Meter): # instrument kinds as non-abstract no-op defaults on the Meter ABC (e.g. # create_gauge in 1.23), which __getattr__ cannot intercept, so new Meter # methods must be audited and overridden here on opentelemetry upgrades. + # tests/contrib/opentelemetry/test_wrapper_abc_drift.py fails when the + # installed opentelemetry-api grows surface not covered here. def __init__(self, meter: Meter) -> None: super().__init__(meter.name, version=meter.version, schema_url=meter.schema_url) self._meter = meter diff --git a/tests/contrib/opentelemetry/test_wrapper_abc_drift.py b/tests/contrib/opentelemetry/test_wrapper_abc_drift.py new file mode 100644 index 000000000..132a4b8f0 --- /dev/null +++ b/tests/contrib/opentelemetry/test_wrapper_abc_drift.py @@ -0,0 +1,111 @@ +"""Drift guard: replay-safe wrappers must override the full public surface of +the OpenTelemetry ABCs they subclass. + +OpenTelemetry adds new API surface to its ABCs as non-abstract no-op defaults +(e.g. ``Meter.create_gauge`` in 1.23, ``Span.add_link`` in 1.23). Because +those defaults exist on the class, ``__getattr__`` delegation cannot intercept +them, so any method a wrapper does not explicitly override silently no-ops +instead of delegating to the wrapped instance -- swallowing telemetry for +every caller. These tests enumerate the installed ABCs by reflection so a new +opentelemetry-api release that grows surface fails loudly (notably in the +latest-dependency CI job) instead of degrading silently. +""" + +import inspect + +import pytest +from opentelemetry._logs import Logger, LoggerProvider +from opentelemetry.metrics import ( + Counter, + Histogram, + Meter, + MeterProvider, + UpDownCounter, + _Gauge, +) +from opentelemetry.trace import Span, Tracer, TracerProvider + +from temporalio.contrib.opentelemetry import ( + ReplaySafeLoggerProvider, + ReplaySafeMeterProvider, + ReplaySafeTracerProvider, +) +from temporalio.contrib.opentelemetry._logger_provider import _ReplaySafeLogger +from temporalio.contrib.opentelemetry._meter_provider import ( + _ReplaySafeCounter, + _ReplaySafeGauge, + _ReplaySafeHistogram, + _ReplaySafeMeter, + _ReplaySafeUpDownCounter, +) +from temporalio.contrib.opentelemetry._tracer_provider import ( + _ReplaySafeSpan, + _ReplaySafeTracer, +) + + +def _public_surface(abc: type) -> set[str]: + """Method names a wrapper must cover: every public callable attribute of + the ABC (which includes non-abstract no-op defaults) plus every non-dunder + abstract method. Properties are excluded: they carry instrumentation-scope + metadata, not telemetry, and inheriting them is safe.""" + surface = { + name + for name in dir(abc) + if not name.startswith("_") and callable(inspect.getattr_static(abc, name)) + } + surface.update( + name + for name in getattr(abc, "__abstractmethods__", ()) + if not name.startswith("__") + ) + return surface + + +# (wrapper, wrapped OTel ABC, intentionally inherited members). An entry in +# the third element is an explicit opt-out: add a name there only when +# inheriting the OTel default is deliberate, and say why in a comment. +_WRAPPER_CASES: list[tuple[type, type, frozenset[str]]] = [ + (_ReplaySafeMeter, Meter, frozenset()), + (ReplaySafeMeterProvider, MeterProvider, frozenset()), + (_ReplaySafeCounter, Counter, frozenset()), + (_ReplaySafeUpDownCounter, UpDownCounter, frozenset()), + (_ReplaySafeHistogram, Histogram, frozenset()), + (_ReplaySafeGauge, _Gauge, frozenset()), + (_ReplaySafeLogger, Logger, frozenset()), + (ReplaySafeLoggerProvider, LoggerProvider, frozenset()), + (_ReplaySafeTracer, Tracer, frozenset()), + (ReplaySafeTracerProvider, TracerProvider, frozenset()), + (_ReplaySafeSpan, Span, frozenset()), +] + + +@pytest.mark.parametrize( + ("wrapper", "abc", "intentionally_inherited"), + _WRAPPER_CASES, + ids=[wrapper.__name__ for wrapper, _, _ in _WRAPPER_CASES], +) +def test_wrapper_overrides_full_otel_abc_surface( + wrapper: type, abc: type, intentionally_inherited: frozenset[str] +): + surface = _public_surface(abc) + assert surface, ( + f"reflection found no public methods on {abc.__module__}.{abc.__qualname__};" + " the drift guard would be vacuous" + ) + + missing = surface - set(vars(wrapper)) - intentionally_inherited + assert not missing, ( + f"{wrapper.__name__} inherits {sorted(missing)} from" + f" {abc.__module__}.{abc.__qualname__} without overriding them. Inherited" + " OTel defaults no-op instead of delegating to the wrapped instance," + " silently dropping telemetry for all callers. Override each method" + " (delegate, wrap, or replay-gate as appropriate) or add it to this" + " test's intentionally-inherited allowlist with a comment." + ) + + stale = intentionally_inherited - surface + assert not stale, ( + f"allowlist entries {sorted(stale)} for {wrapper.__name__} are not on" + f" {abc.__module__}.{abc.__qualname__} anymore; remove them" + ) From 373ff335ff176854e36dfe2f8a0f335c857dd1eb Mon Sep 17 00:00:00 2001 From: DABH Date: Tue, 11 Aug 2026 16:01:24 -0500 Subject: [PATCH 24/24] Harden OTel global provider isolation in tests The OTel globals are set-once per process and xdist worksteal ignores xdist_group pinning, so tests share workers with arbitrary siblings. Assert that global provider installs take effect, park proxy meters on a no-op provider around the meter-reset fixture, and uninstrument GoogleADKInstrumentor after test_single_agent_telemetry. --- tests/conftest.py | 42 +++++++--- .../test_google_adk_agents.py | 76 ++++++++++--------- .../google_adk_agents/test_replay_metrics.py | 60 +++++++++++---- .../openai_agents/test_openai_tracing.py | 8 ++ 4 files changed, 127 insertions(+), 59 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 83c24e4e7..3c02d8d18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,12 @@ from collections.abc import AsyncGenerator, Iterator import opentelemetry._logs._internal +import opentelemetry.metrics import opentelemetry.metrics._internal import opentelemetry.trace import pytest import pytest_asyncio +from opentelemetry.metrics import NoOpMeterProvider from opentelemetry.util._once import Once from temporalio.client import Client @@ -267,12 +269,21 @@ def continue_as_new_suggest_history_count() -> int: # OpenTelemetry's global providers are set-once per process with no public # way to unset them, so tests needing their own provider must reset the # globals directly -- the same isolation pattern OpenTelemetry's own test -# suite uses (opentelemetry.test.globals_test). +# suite uses (opentelemetry.test.globals_test). Isolation cannot be delegated +# to scheduling: CI runs pytest-xdist with --dist=worksteal, which ignores +# xdist_group pinning, so any test in the suite can share a worker process +# with any other. Every test that installs a global provider must therefore +# use these fixtures and leave the globals reset behind it. @pytest.fixture def reset_otel_tracer_provider(): - """Isolate global OpenTelemetry tracer provider state around a test.""" + """Isolate global OpenTelemetry tracer provider state around a test. + + Proxy tracers bound to a real provider stay bound forever; OTel has no + rebind mechanism for tracers. Tests must install their provider before + any span is created through a proxy tracer they care about. + """ opentelemetry.trace._TRACER_PROVIDER_SET_ONCE = Once() opentelemetry.trace._TRACER_PROVIDER = None yield @@ -280,20 +291,31 @@ def reset_otel_tracer_provider(): opentelemetry.trace._TRACER_PROVIDER = None +def _reset_meter_provider_globals() -> None: + # Reset the set-once latch, then park proxy meters on a no-op provider: + # set_meter_provider rebinds every proxy meter and its instruments, so + # instruments bound during an earlier test stop recording into that + # test's dead reader. Reset the latch again so the next installer wins. + opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() + opentelemetry.metrics._internal._METER_PROVIDER = None + opentelemetry.metrics.set_meter_provider(NoOpMeterProvider()) + opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() + opentelemetry.metrics._internal._METER_PROVIDER = None + + @pytest.fixture def reset_otel_meter_provider(): """Isolate global OpenTelemetry meter provider state around a test. - Proxy meters/instruments already bound to a real provider stay bound after - this reset; only the next set_meter_provider call rebinds them. Tests must - not assume an unset global provider drops recordings from instruments - created in earlier tests. + Both setup and teardown park proxy meters on a no-op provider, so + instruments bound by other tests neither record into this test's provider + unexpectedly nor keep recording into this test's reader afterwards. Any + set_meter_provider call rebinds proxy meters, so tests that install their + own provider are unaffected by the parking. """ - opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() - opentelemetry.metrics._internal._METER_PROVIDER = None + _reset_meter_provider_globals() yield - opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once() - opentelemetry.metrics._internal._METER_PROVIDER = None + _reset_meter_provider_globals() @pytest.fixture diff --git a/tests/contrib/google_adk_agents/test_google_adk_agents.py b/tests/contrib/google_adk_agents/test_google_adk_agents.py index 2a1cf6aa1..0c073f40c 100644 --- a/tests/contrib/google_adk_agents/test_google_adk_agents.py +++ b/tests/contrib/google_adk_agents/test_google_adk_agents.py @@ -538,45 +538,51 @@ async def test_single_agent_telemetry( provider = create_tracer_provider() provider.add_span_processor(SimpleSpanProcessor(exporter)) set_tracer_provider(provider) + # Instrumentors are process-global singletons bound to the provider seen + # at instrument() time; without uninstrument() ADK code in every later + # test in this process would keep emitting spans into this test's + # provider, and re-instrumentation attempts would silently no-op. GoogleADKInstrumentor().instrument() + try: + new_config = client.config() + new_config["plugins"] = [ + GoogleAdkPlugin(), + OpenTelemetryPlugin(add_temporal_spans=True), + ] + client = Client(**new_config) - new_config = client.config() - new_config["plugins"] = [ - GoogleAdkPlugin(), - OpenTelemetryPlugin(add_temporal_spans=True), - ] - client = Client(**new_config) - - # Run Worker with the ADK plugin - async with Worker( - client, - task_queue="adk-task-queue-telemetry", - activities=[ - get_weather, - ], - workflows=[WeatherAgent], - max_cached_workflows=0, - ): - LLMRegistry.register(WeatherModel) - - # Test Weather Agent - handle = await client.start_workflow( - WeatherAgent.run, - args=[ - "What is the weather in New York?", - "weather_model", - ], - id=f"weather-agent-telemetry-workflow-{uuid.uuid4()}", + # Run Worker with the ADK plugin + async with Worker( + client, task_queue="adk-task-queue-telemetry", - execution_timeout=timedelta(seconds=60), - ) - result = await handle.result() - print(f"Workflow result: {result}") + activities=[ + get_weather, + ], + workflows=[WeatherAgent], + max_cached_workflows=0, + ): + LLMRegistry.register(WeatherModel) + + # Test Weather Agent + handle = await client.start_workflow( + WeatherAgent.run, + args=[ + "What is the weather in New York?", + "weather_model", + ], + id=f"weather-agent-telemetry-workflow-{uuid.uuid4()}", + task_queue="adk-task-queue-telemetry", + execution_timeout=timedelta(seconds=60), + ) + result = await handle.result() + print(f"Workflow result: {result}") - assert result is not None - assert result.content is not None - assert result.content.parts is not None - assert result.content.parts[0].text == "warm and sunny" + assert result is not None + assert result.content is not None + assert result.content.parts is not None + assert result.content.parts[0].text == "warm and sunny" + finally: + GoogleADKInstrumentor().uninstrument() print("\n".join(dump_spans(exporter.get_finished_spans(), with_attributes=False))) assert dump_spans(exporter.get_finished_spans(), with_attributes=False) == [ diff --git a/tests/contrib/google_adk_agents/test_replay_metrics.py b/tests/contrib/google_adk_agents/test_replay_metrics.py index dc3ffa68a..689e58820 100644 --- a/tests/contrib/google_adk_agents/test_replay_metrics.py +++ b/tests/contrib/google_adk_agents/test_replay_metrics.py @@ -27,6 +27,7 @@ from google.adk.runners import InMemoryRunner from google.adk.utils.context_utils import Aclosing from google.genai import types +from opentelemetry._logs import LoggerProvider as ApiLoggerProvider from opentelemetry.metrics import Meter from opentelemetry.metrics import MeterProvider as ApiMeterProvider from opentelemetry.sdk._logs import LoggerProvider @@ -37,7 +38,8 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.trace import set_tracer_provider +from opentelemetry.trace import TracerProvider as ApiTracerProvider +from opentelemetry.trace import get_tracer_provider, set_tracer_provider from opentelemetry.util.types import Attributes import temporalio.contrib.google_adk_agents.workflow @@ -76,6 +78,36 @@ "gen_ai.choice": 2, } +# The OTel globals are set-once per process and these tests share xdist +# workers with arbitrary sibling tests, so every install asserts it took +# effect: a leaked global from a sibling would otherwise surface as +# mysteriously failing count assertions here (and vice versa). + + +def _install_global_meter_provider(provider: ApiMeterProvider) -> None: + opentelemetry.metrics.set_meter_provider(provider) + assert opentelemetry.metrics.get_meter_provider() is provider, ( + "Global meter provider install was a no-op; a previous test in this" + " process left a provider set without resetting it" + ) + + +def _install_global_tracer_provider(provider: ApiTracerProvider) -> None: + set_tracer_provider(provider) + assert get_tracer_provider() is provider, ( + "Global tracer provider install was a no-op; a previous test in this" + " process left a provider set without resetting it" + ) + + +def _install_global_logger_provider(provider: ApiLoggerProvider) -> None: + opentelemetry._logs.set_logger_provider(provider) + assert opentelemetry._logs.get_logger_provider() is provider, ( + "Global logger provider install was a no-op; a previous test in this" + " process left a provider set without resetting it" + ) + + # Counts real (worker-side) activity executions; replays must not add to it. activity_executions = 0 @@ -290,7 +322,7 @@ async def test_replay_safe_meter_provider_suppresses_replay_metrics( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] ): reader = InMemoryMetricReader() - opentelemetry.metrics.set_meter_provider( + _install_global_meter_provider( ReplaySafeMeterProvider(MeterProvider(metric_readers=[reader])) ) @@ -309,7 +341,7 @@ async def test_replay_metrics_duplicate_without_replay_safe_meter_provider( # Control: without the wrapper, every replay re-records every # workflow-side ADK metric even though nothing really re-executed. reader = InMemoryMetricReader() - opentelemetry.metrics.set_meter_provider(MeterProvider(metric_readers=[reader])) + _install_global_meter_provider(MeterProvider(metric_readers=[reader])) real_executions = await _run_once_and_replay(client, num_replays=3) @@ -332,7 +364,7 @@ async def test_replay_safe_logger_provider_suppresses_replay_log_events( reset_adk_proxy_logger, # type: ignore[reportUnusedParameter] ): provider, exporter = _in_memory_logger_provider() - opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(provider)) + _install_global_logger_provider(ReplaySafeLoggerProvider(provider)) real_executions = await _run_once_and_replay(client, num_replays=3) @@ -350,7 +382,7 @@ async def test_replay_log_events_duplicate_without_replay_safe_logger_provider( # Control: without the wrapper, every replay re-emits every workflow-side # ADK log event even though nothing really re-executed. provider, exporter = _in_memory_logger_provider() - opentelemetry._logs.set_logger_provider(provider) + _install_global_logger_provider(provider) real_executions = await _run_once_and_replay(client, num_replays=3) @@ -367,7 +399,7 @@ def _worker_config() -> WorkerConfig: def test_plugin_warns_on_non_replay_safe_meter_provider( reset_otel_meter_provider, # type: ignore[reportUnusedParameter] ): - opentelemetry.metrics.set_meter_provider(MeterProvider()) + _install_global_meter_provider(MeterProvider()) with pytest.warns(UserWarning, match="MeterProvider is not replay-safe"): GoogleAdkPlugin().configure_worker(_worker_config()) @@ -375,7 +407,7 @@ def test_plugin_warns_on_non_replay_safe_meter_provider( def test_plugin_warns_on_non_replay_safe_tracer_provider( reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] ): - set_tracer_provider(TracerProvider()) + _install_global_tracer_provider(TracerProvider()) with pytest.warns(UserWarning, match="TracerProvider is not replay-safe"): GoogleAdkPlugin().configure_worker(_worker_config()) @@ -385,7 +417,7 @@ def test_plugin_warns_on_replayer_construction( ): # Replayer replays are exactly where an unsafe global provider re-records # telemetry, so the warning must fire there too. - opentelemetry.metrics.set_meter_provider(MeterProvider()) + _install_global_meter_provider(MeterProvider()) with pytest.warns(UserWarning, match="MeterProvider is not replay-safe"): Replayer(workflows=[ReplayMetricsAgent], plugins=[GoogleAdkPlugin()]) @@ -395,9 +427,9 @@ def test_plugin_does_not_warn_with_replay_safe_providers( reset_otel_tracer_provider, # type: ignore[reportUnusedParameter] reset_otel_logger_provider, # type: ignore[reportUnusedParameter] ): - opentelemetry.metrics.set_meter_provider(ReplaySafeMeterProvider(MeterProvider())) - set_tracer_provider(create_tracer_provider()) - opentelemetry._logs.set_logger_provider(ReplaySafeLoggerProvider(LoggerProvider())) + _install_global_meter_provider(ReplaySafeMeterProvider(MeterProvider())) + _install_global_tracer_provider(create_tracer_provider()) + _install_global_logger_provider(ReplaySafeLoggerProvider(LoggerProvider())) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") GoogleAdkPlugin().configure_worker(_worker_config()) @@ -438,7 +470,7 @@ def test_plugin_does_not_warn_on_unknown_custom_provider( # A custom provider delegating to a replay-safe one is a fully replay-safe # configuration; an unclassifiable provider must not trigger a false # positive. - opentelemetry.metrics.set_meter_provider(_DelegatingMeterProvider()) + _install_global_meter_provider(_DelegatingMeterProvider()) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") GoogleAdkPlugin().configure_worker(_worker_config()) @@ -452,7 +484,7 @@ async def test_plugin_warning_points_at_worker_construction( ): # stacklevel on the warning must attribute it to the user's Worker(...) # call, i.e. this file, not SDK internals. - opentelemetry.metrics.set_meter_provider(MeterProvider()) + _install_global_meter_provider(MeterProvider()) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") Worker( @@ -487,7 +519,7 @@ def test_plugin_warning_points_at_wrapping_plugin_caller( # When another plugin wraps GoogleAdkPlugin, the warning must attribute # to the nearest user frame (the wrapper's delegation line), not SDK # internals or a fixed frame depth. - opentelemetry.metrics.set_meter_provider(MeterProvider()) + _install_global_meter_provider(MeterProvider()) with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") _WrappingPlugin().configure_worker(_worker_config()) diff --git a/tests/contrib/openai_agents/test_openai_tracing.py b/tests/contrib/openai_agents/test_openai_tracing.py index facc3212b..1ee3b01ec 100644 --- a/tests/contrib/openai_agents/test_openai_tracing.py +++ b/tests/contrib/openai_agents/test_openai_tracing.py @@ -292,6 +292,14 @@ def set_test_tracer_provider() -> InMemorySpanExporter: provider = create_tracer_provider() provider.add_span_processor(SimpleSpanProcessor(exporter)) opentelemetry.trace.set_tracer_provider(provider) + # set_tracer_provider is set-once per process: if another test left a + # global provider installed (e.g. leaked from a sibling test in the same + # pytest-xdist worker), the call above silently no-ops and every span in + # this test bypasses the exporter. Fail at the cause instead. + assert opentelemetry.trace.get_tracer_provider() is provider, ( + "Global tracer provider install was a no-op; a previous test in this" + " process left a provider set without resetting it" + ) return exporter