From a66d66d7852cdd762b01e92dbae20149213329c3 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 4 Sep 2026 16:33:12 -0400 Subject: [PATCH 1/5] Propagate LLM Observability context across service boundaries LLMObs context (ml_app, session_id, agent attribution) stayed within a single process. An agent that dispatched work over SQS, or called another service over HTTP, left the downstream side with no session and no agent attribution, fragmenting what is logically one LLM trace. Carry these as _dd.p.llmobs_* propagation tags, using the key names dd-trace-py/js/go already use so a mixed-language pipeline joins up. Rather than teaching each integration about LLMObs, register an LLMObsContextPropagator as a propagation concern: it contributes no headers of its own, it stages the tags onto the span context ahead of the tracing propagator, which then serializes them like any other propagation tag. This mirrors dd-trace-py, where LLMObs subscribes to the generic http.span_inject hook, and means every boundary automatic instrumentation already covers is handled at once. SQS needs no integration-specific code as a result. SqsInterceptor already injects through the default propagator, and the consume span is active while the consumer's per-message code runs, so a worker's LLMObs spans inherit the upstream context. Values are resolved from the ambient LLMObsContext at injection time, so the innermost active span wins and leaving a scope stops contributing. On the receive side, DDLLMObsSpan reads session_id and agent attribution off the propagated context whenever no same-trace in-process parent contributed them -- including when a stale context from an unrelated trace is present, which must not suppress attribution that legitimately arrived over the wire. --- .../trace/llmobs/LLMObsContextPropagator.java | 62 ++++++ .../datadog/trace/llmobs/LLMObsSystem.java | 6 + .../trace/llmobs/domain/DDLLMObsSpan.java | 25 +++ .../llmobs/LLMObsContextPropagatorTest.java | 197 ++++++++++++++++++ .../datadog/trace/core/DDSpanContext.java | 40 ++++ .../core/propagation/ExtractedContext.java | 20 ++ .../core/propagation/PropagationTags.java | 36 ++++ .../propagation/ptags/DatadogPTagsCodec.java | 19 +- .../propagation/ptags/LLMObsTagValues.java | 23 ++ .../core/propagation/ptags/PTagsCodec.java | 43 ++++ .../core/propagation/ptags/PTagsFactory.java | 131 +++++++++++- .../core/propagation/ptags/W3CPTagsCodec.java | 26 ++- .../trace/api/llmobs/LLMObsContext.java | 35 ++++ .../instrumentation/api/AgentPropagation.java | 5 + .../instrumentation/api/AgentSpanContext.java | 52 +++++ 15 files changed, 710 insertions(+), 10 deletions(-) create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java create mode 100644 dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java create mode 100644 dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java new file mode 100644 index 00000000000..92d6f3e0e26 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java @@ -0,0 +1,62 @@ +package datadog.trace.llmobs; + +import datadog.context.Context; +import datadog.context.propagation.CarrierSetter; +import datadog.context.propagation.CarrierVisitor; +import datadog.context.propagation.Propagator; +import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; + +/** + * Stages the LLM Observability propagation tags onto the span context being injected, so that every + * boundary already covered by automatic instrumentation — HTTP, gRPC, SQS, Kafka, ... — carries + * LLMObs context without the application having to propagate it by hand. + * + *

This propagator writes nothing to the carrier itself. It runs ahead of the tracing propagator + * (see {@code AgentPropagation.LLMOBS_CONCERN}) and only populates the {@code _dd.p.llmobs_*} + * fields on the span context; the tracing propagator then serializes them into {@code + * x-datadog-tags} / {@code tracestate} along with every other propagation tag. This mirrors + * dd-trace-py, where LLMObs subscribes to the generic {@code http.span_inject} hook that {@code + * HTTPPropagator.inject} fires on every outbound request, rather than owning a separate wire + * format. + * + *

Values are resolved from the ambient {@link LLMObsContext} at injection time rather than being + * written once when a span starts. That way the innermost active LLMObs span always wins, and + * leaving an LLMObs scope stops contributing its tags without any save/restore bookkeeping. + */ +public class LLMObsContextPropagator implements Propagator { + + @Override + public void inject(Context context, C carrier, CarrierSetter setter) { + AgentSpan span = AgentSpan.fromContext(context); + if (span == null) { + return; + } + AgentSpanContext spanContext = span.spanContext(); + if (spanContext == null) { + return; + } + + // Gate on trace-id consistency, the same way DDLLMObsSpan gates parent_id/session_id + // inheritance. An LLMObs context leaked across an async boundary must not tag an outbound + // request that belongs to an unrelated trace. + AgentSpanContext llmObsContext = LLMObsContext.current(); + if (llmObsContext == null || llmObsContext.getTraceId() != spanContext.getTraceId()) { + return; + } + + spanContext.updateLLMObsMlApp(LLMObsContext.currentMlApp()); + spanContext.updateLLMObsSessionId(LLMObsContext.currentSessionId()); + spanContext.updateLLMObsParentAgentSpanId(LLMObsContext.currentParentAgentSpanId()); + spanContext.updateLLMObsParentAgentName(LLMObsContext.currentParentAgentName()); + } + + @Override + public Context extract(Context context, C carrier, CarrierVisitor visitor) { + // Nothing to do: the tracing propagator's codecs already parse the _dd.p.llmobs_* tags back + // into the extracted context's propagation tags, and DDLLMObsSpan reads them from there when + // no in-process LLMObs parent applies. + return context; + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 864cf27eb2c..a819598b14d 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -1,6 +1,7 @@ package datadog.trace.llmobs; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.context.propagation.Propagators; import datadog.trace.api.Config; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -8,6 +9,7 @@ import datadog.trace.api.llmobs.LLMObsSpan; import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.api.telemetry.LLMObsMetricCollector; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.llmobs.domain.DDLLMObsSpan; import datadog.trace.llmobs.domain.LLMObsEval; @@ -51,6 +53,10 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { LLMObsInternal.setEvalProcessor(new LLMObsCustomEvalProcessor(mlApp, sco, config)); LLMObsInternal.setFeedbackProcessor(new LLMObsCustomFeedbackProcessor(mlApp, sco, config)); + + // Carry LLMObs context across every boundary automatic instrumentation already covers, by + // staging the _dd.p.llmobs_* tags on each injected span context. See LLMObsContextPropagator. + Propagators.register(AgentPropagation.LLMOBS_CONCERN, new LLMObsContextPropagator()); } private static class LLMObsCustomFeedbackProcessor implements LLMObs.LLMObsFeedbackProcessor { diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 473d118cdc3..7dd69393a0f 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -159,6 +159,7 @@ public DDLLMObsSpan( String samplingDecision = null; String resolvedParentAgentSpanId = null; String resolvedParentAgentName = null; + boolean inheritedInProcess = false; if (null != parent) { if (parent.getTraceId() != span.getTraceId()) { LOGGER.error( @@ -168,6 +169,7 @@ public DDLLMObsSpan( span.getTraceId(), span.getSpanId()); } else { + inheritedInProcess = true; parentSpanID = String.valueOf(parent.getSpanId()); // Inherit session_id from parent context only when it belongs to the same trace. // Matches dd-trace-py and dd-trace-js: session_id need only be set on the root @@ -197,6 +199,23 @@ public DDLLMObsSpan( } } + if (!inheritedInProcess) { + // No usable in-process LLMObs parent, but this span may still be continuing a trace that + // arrived from another service — an SQS worker handling a message, an inbound HTTP request. + // The upstream values are on the span context's propagation tags, parsed back out of + // x-datadog-tags / tracestate by the tracing propagator. + // + // This also covers the trace-mismatch branch above: a stale context leaked from an unrelated + // trace must not suppress attribution that legitimately arrived over the wire. + if (sessionId == null || sessionId.isEmpty()) { + sessionId = asString(span.spanContext().getLLMObsSessionId()); + } + resolvedParentAgentSpanId = asString(span.spanContext().getLLMObsParentAgentSpanId()); + if (resolvedParentAgentSpanId != null) { + resolvedParentAgentName = asString(span.spanContext().getLLMObsParentAgentName()); + } + } + // An agent span is its own descendants' nearest agent ancestor, replacing anything inherited. // Use the span name as the initial pagent name; annotateAgentManifest() will update it to the // manifest name if one is provided later. @@ -236,6 +255,7 @@ public DDLLMObsSpan( scope = LLMObsContext.attach( span.spanContext(), + mlApp, sessionId, resolvedAgentVersion, sampleRate, @@ -717,4 +737,9 @@ public DDTraceId getTraceId() { public long getSpanId() { return span.getSpanId(); } + + /** Narrow a propagated tag value to a non-empty String, or null. */ + private static String asString(CharSequence value) { + return value == null || value.length() == 0 ? null : value.toString(); + } } diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java new file mode 100644 index 00000000000..cbde80af5ba --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java @@ -0,0 +1,197 @@ +package datadog.trace.llmobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.context.Context; +import datadog.context.propagation.Propagators; +import datadog.trace.agent.tooling.TracerInstaller; +import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreTracer; +import datadog.trace.llmobs.domain.DDLLMObsSpan; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Covers automatic LLM Observability context propagation: no LLMObs propagation API is called + * anywhere in these tests. Injecting the active span the way auto-instrumentation does — an HTTP + * client, or the SQS interceptor writing message attributes — must carry the LLMObs context. + * + *

The carrier is a plain {@code Map}, which is the shape both an HTTP header map + * and the SQS {@code _datadog} message attribute reduce to at the propagator boundary. + */ +class LLMObsContextPropagatorTest { + + private static final String ML_APP_TAG = "_dd.p.llmobs_ml_app"; + private static final String SESSION_ID_TAG = "_dd.p.llmobs_sid"; + private static final String PAGENT_SPAN_ID_TAG = "_dd.p.llmobs_pagent_span_id"; + private static final String PAGENT_NAME_TAG = "_dd.p.llmobs_pagent_name"; + + private static CoreTracer tracer; + + @BeforeAll + static void installTracer() { + tracer = CoreTracer.builder().build(); + TracerInstaller.forceInstallGlobalTracer(tracer); + Propagators.register(AgentPropagation.LLMOBS_CONCERN, new LLMObsContextPropagator()); + } + + @AfterAll + static void closeTracer() { + TracerInstaller.forceInstallGlobalTracer(null); + tracer.close(); + } + + private static DDLLMObsSpan newSpan(String kind, String name, String mlApp, String sessionId) { + WellKnownTags tags = + new WellKnownTags("runtime-id", "hostname", "test", "service", "version", "java"); + return new DDLLMObsSpan(kind, name, mlApp, sessionId, "service", tags); + } + + private static AgentScope startRootApmScope() { + AgentSpan root = AgentTracer.get().buildSpan("apm", "sqs.produce").start(); + return AgentTracer.activateSpan(root); + } + + /** What an auto-instrumented client does: inject the active span into an outbound carrier. */ + private static Map autoInject(AgentSpan span) { + Map carrier = new HashMap<>(); + Propagators.defaultPropagator().inject(span, carrier, Map::put); + return carrier; + } + + @Test + void stagesLlmObsTagsOnInjectionWithoutAnyManualPropagation() { + Map carrier; + String agentSpanId; + try (AgentScope apmScope = startRootApmScope()) { + DDLLMObsSpan agent = newSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "planner", "my-ml-app", "sess-1"); + agentSpanId = String.valueOf(agent.getSpanId()); + try { + carrier = autoInject(AgentTracer.activeSpan()); + } finally { + agent.finish(); + } + } + + String tags = carrier.get("x-datadog-tags"); + assertNotNull(tags, "expected x-datadog-tags to be injected"); + assertTrue(tags.contains(ML_APP_TAG + "=my-ml-app"), () -> "ml_app missing from " + tags); + assertTrue(tags.contains(SESSION_ID_TAG + "=sess-1"), () -> "session_id missing from " + tags); + assertTrue( + tags.contains(PAGENT_SPAN_ID_TAG + "=" + agentSpanId), + () -> "pagent_span_id missing from " + tags); + assertTrue( + tags.contains(PAGENT_NAME_TAG + "=planner"), () -> "pagent_name missing from " + tags); + } + + @Test + void addsNothingWhenNoLlmObsSpanIsActive() { + Map carrier; + try (AgentScope apmScope = startRootApmScope()) { + carrier = autoInject(apmScope.span()); + } + + String tags = carrier.get("x-datadog-tags"); + assertTrue( + tags == null || !tags.contains("_dd.p.llmobs_"), () -> "unexpected LLMObs tags in " + tags); + } + + @Test + void stopsContributingTagsOnceTheLlmObsScopeIsClosed() { + Map carrier; + try (AgentScope apmScope = startRootApmScope()) { + newSpan(Tags.LLMOBS_WORKFLOW_SPAN_KIND, "work", "my-ml-app", "sess-1").finish(); + // The LLMObs span has finished; a later outbound call on the same APM trace must not be + // tagged with a session that is no longer active. + carrier = autoInject(apmScope.span()); + } + + String tags = carrier.get("x-datadog-tags"); + assertTrue( + tags == null || !tags.contains(SESSION_ID_TAG), + () -> "session_id leaked after scope close: " + tags); + } + + /** + * The full cross-process hop, as an SQS producer/worker pair sees it: the producer injects into + * message attributes, the worker extracts and activates them, and an LLMObs span started by the + * worker inherits the session and agent attribution without any application-level plumbing. + */ + @Test + void workerInheritsSessionAndAgentAttributionAcrossTheBoundary() { + Map messageAttributes; + long producerTraceId; + String producerAgentSpanId; + + try (AgentScope apmScope = startRootApmScope()) { + DDLLMObsSpan producer = + newSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "dispatcher", "my-ml-app", "sess-42"); + producerTraceId = producer.getTraceId().toLong(); + producerAgentSpanId = String.valueOf(producer.getSpanId()); + try { + messageAttributes = autoInject(AgentTracer.activeSpan()); + } finally { + producer.finish(); + } + } + + // Worker side: a fresh context, as a message handler would have. + Context extracted = + Propagators.defaultPropagator() + .extract( + Context.root(), messageAttributes, (carrier, visitor) -> carrier.forEach(visitor)); + AgentSpan consumeSpan = AgentSpan.fromContext(extracted); + assertNotNull(consumeSpan, "expected trace context to be extracted"); + + try (AgentScope consumeScope = AgentTracer.get().activateSpan(consumeSpan)) { + DDLLMObsSpan workerTool = newSpan(Tags.LLMOBS_TOOL_SPAN_KIND, "handler", "my-ml-app", null); + try { + assertEquals(producerTraceId, workerTool.getTraceId().toLong(), "trace should be joined"); + // The span publishes its resolved values to the context for its own descendants, so this + // is what the worker's LLMObs span actually settled on. + assertEquals("sess-42", LLMObsContext.currentSessionId()); + assertEquals(producerAgentSpanId, LLMObsContext.currentParentAgentSpanId()); + assertEquals("dispatcher", LLMObsContext.currentParentAgentName()); + } finally { + workerTool.finish(); + } + } + } + + @Test + void workerWithoutUpstreamLlmObsContextInheritsNothing() { + Map messageAttributes; + try (AgentScope apmScope = startRootApmScope()) { + messageAttributes = autoInject(apmScope.span()); + } + + Context extracted = + Propagators.defaultPropagator() + .extract( + Context.root(), messageAttributes, (carrier, visitor) -> carrier.forEach(visitor)); + AgentSpan consumeSpan = AgentSpan.fromContext(extracted); + assertNotNull(consumeSpan, "expected trace context to be extracted"); + + try (AgentScope consumeScope = AgentTracer.get().activateSpan(consumeSpan)) { + DDLLMObsSpan workerTool = newSpan(Tags.LLMOBS_TOOL_SPAN_KIND, "handler", "my-ml-app", null); + try { + assertNull(LLMObsContext.currentSessionId()); + assertNull(LLMObsContext.currentParentAgentSpanId()); + } finally { + workerTool.finish(); + } + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index adf4cd66156..73b3e2b24e1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -1492,6 +1492,46 @@ public PropagationTags getPropagationTags() { return getRootSpanContextOrThis().propagationTags; } + @Override + public CharSequence getLLMObsMlApp() { + return getPropagationTags().getLLMObsMlApp(); + } + + @Override + public void updateLLMObsMlApp(CharSequence mlApp) { + getPropagationTags().updateLLMObsMlApp(mlApp); + } + + @Override + public CharSequence getLLMObsSessionId() { + return getPropagationTags().getLLMObsSessionId(); + } + + @Override + public void updateLLMObsSessionId(CharSequence sessionId) { + getPropagationTags().updateLLMObsSessionId(sessionId); + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return getPropagationTags().getLLMObsParentAgentSpanId(); + } + + @Override + public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { + getPropagationTags().updateLLMObsParentAgentSpanId(parentAgentSpanId); + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return getPropagationTags().getLLMObsParentAgentName(); + } + + @Override + public void updateLLMObsParentAgentName(CharSequence parentAgentName) { + getPropagationTags().updateLLMObsParentAgentName(parentAgentName); + } + /** TraceSegment Implementation */ @Override public void setTagTop(String key, Object value, boolean sanitize) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java index af503e6a6ed..52a40a94e4c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java @@ -117,6 +117,26 @@ public PropagationTags getPropagationTags() { return propagationTags; } + @Override + public CharSequence getLLMObsMlApp() { + return propagationTags.getLLMObsMlApp(); + } + + @Override + public CharSequence getLLMObsSessionId() { + return propagationTags.getLLMObsSessionId(); + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return propagationTags.getLLMObsParentAgentSpanId(); + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return propagationTags.getLLMObsParentAgentName(); + } + @Override public String toString() { StringBuilder builder = new StringBuilder("ExtractedContext{"); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java index 3a0c57a4dd8..47161ef1276 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java @@ -169,6 +169,42 @@ public interface Factory { */ public abstract void updateOrgPropagationMarker(CharSequence opm); + /** + * Returns the LLM Observability {@code ml_app} currently propagated with this trace, encoded as + * {@code _dd.p.llmobs_ml_app}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsMlApp(); + + /** Sets the LLM Observability {@code ml_app} to propagate with this trace. */ + public abstract void updateLLMObsMlApp(CharSequence mlApp); + + /** + * Returns the LLM Observability {@code session_id} currently propagated with this trace, encoded + * as {@code _dd.p.llmobs_sid}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsSessionId(); + + /** Sets the LLM Observability {@code session_id} to propagate with this trace. */ + public abstract void updateLLMObsSessionId(CharSequence sessionId); + + /** + * Returns the span id of the parent LLM Observability agent span currently propagated with this + * trace, encoded as {@code _dd.p.llmobs_pagent_span_id}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsParentAgentSpanId(); + + /** Sets the parent LLM Observability agent span id to propagate with this trace. */ + public abstract void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId); + + /** + * Returns the name of the parent LLM Observability agent span currently propagated with this + * trace, encoded as {@code _dd.p.llmobs_pagent_name}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsParentAgentName(); + + /** Sets the parent LLM Observability agent span name to propagate with this trace. */ + public abstract void updateLLMObsParentAgentName(CharSequence parentAgentName); + public HashMap createTagMap() { HashMap result = new HashMap<>(); fillTagMap(result); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index 3ac0c7ad712..907fab25e36 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -64,6 +64,10 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { TagValue traceIdTagValue = null; int traceSource = 0; TagValue orgPropagationMarkerTagValue = null; + TagValue llmObsMlAppTagValue = null; + TagValue llmObsSessionIdTagValue = null; + TagValue llmObsParentAgentSpanIdTagValue = null; + TagValue llmObsParentAgentNameTagValue = null; while (tagPos < len) { int tagKeyEndsAt = validateCharsUntilSeparatorOrEnd( @@ -102,6 +106,14 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { traceSource = ProductTraceSource.parseBitfieldHex(tagValue.toString()); } else if (tagKey.equals(ORG_PROPAGATION_MARKER_TAG)) { orgPropagationMarkerTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_ML_APP_TAG)) { + llmObsMlAppTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_SESSION_ID_TAG)) { + llmObsSessionIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_SPAN_ID_TAG)) { + llmObsParentAgentSpanIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { + llmObsParentAgentNameTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -119,7 +131,12 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { decisionMakerTagValue, traceIdTagValue, traceSource, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + new LLMObsTagValues( + llmObsMlAppTagValue, + llmObsSessionIdTagValue, + llmObsParentAgentSpanIdTagValue, + llmObsParentAgentNameTagValue)); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java new file mode 100644 index 00000000000..7d34fdab011 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java @@ -0,0 +1,23 @@ +package datadog.trace.core.propagation.ptags; + +/** + * Bundles the four LLM Observability propagation tag values ({@code ml_app}, {@code session_id}, + * parent agent span id, parent agent name) extracted from an incoming header, so they can be + * threaded through {@link PTagsFactory.PTags} construction as a single parameter. + */ +final class LLMObsTagValues { + static final LLMObsTagValues EMPTY = new LLMObsTagValues(null, null, null, null); + + final TagValue mlApp; + final TagValue sessionId; + final TagValue parentAgentSpanId; + final TagValue parentAgentName; + + LLMObsTagValues( + TagValue mlApp, TagValue sessionId, TagValue parentAgentSpanId, TagValue parentAgentName) { + this.mlApp = mlApp; + this.sessionId = sessionId; + this.parentAgentSpanId = parentAgentSpanId; + this.parentAgentName = parentAgentName; + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index e2c0658a1d2..99875e7f0a3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -23,6 +23,10 @@ abstract class PTagsCodec { protected static final String PROPAGATION_ERROR_MALFORMED_TID = "malformed_tid "; protected static final String PROPAGATION_ERROR_INCONSISTENT_TID = "inconsistent_tid "; protected static final TagKey UPSTREAM_SERVICES_DEPRECATED_TAG = TagKey.from("upstream_services"); + protected static final TagKey LLMOBS_ML_APP_TAG = TagKey.from("llmobs_ml_app"); + protected static final TagKey LLMOBS_SESSION_ID_TAG = TagKey.from("llmobs_sid"); + protected static final TagKey LLMOBS_PAGENT_SPAN_ID_TAG = TagKey.from("llmobs_pagent_span_id"); + protected static final TagKey LLMOBS_PAGENT_NAME_TAG = TagKey.from("llmobs_pagent_name"); static String headerValue(PTagsCodec codec, PTags ptags) { return headerValue(codec, ptags, null); @@ -65,6 +69,22 @@ static String headerValue(PTagsCodec codec, PTags ptags, CharSequence lastParent codec.appendTag( sb, ORG_PROPAGATION_MARKER_TAG, ptags.getOrgPropagationMarkerTagValue(), size); } + if (ptags.getLLMObsMlAppTagValue() != null) { + size = codec.appendTag(sb, LLMOBS_ML_APP_TAG, ptags.getLLMObsMlAppTagValue(), size); + } + if (ptags.getLLMObsSessionIdTagValue() != null) { + size = codec.appendTag(sb, LLMOBS_SESSION_ID_TAG, ptags.getLLMObsSessionIdTagValue(), size); + } + if (ptags.getLLMObsParentAgentSpanIdTagValue() != null) { + size = + codec.appendTag( + sb, LLMOBS_PAGENT_SPAN_ID_TAG, ptags.getLLMObsParentAgentSpanIdTagValue(), size); + } + if (ptags.getLLMObsParentAgentNameTagValue() != null) { + size = + codec.appendTag( + sb, LLMOBS_PAGENT_NAME_TAG, ptags.getLLMObsParentAgentNameTagValue(), size); + } Iterator it = ptags.getTagPairs().iterator(); while (it.hasNext() && !codec.isTooLarge(sb, size)) { TagElement tagKey = it.next(); @@ -137,6 +157,29 @@ static void fillTagMap(PTags propagationTags, Map tagMap) { .forType(Encoding.DATADOG) .toString()); } + if (propagationTags.getLLMObsMlAppTagValue() != null) { + tagMap.put( + LLMOBS_ML_APP_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsMlAppTagValue().forType(Encoding.DATADOG).toString()); + } + if (propagationTags.getLLMObsSessionIdTagValue() != null) { + tagMap.put( + LLMOBS_SESSION_ID_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsSessionIdTagValue().forType(Encoding.DATADOG).toString()); + } + if (propagationTags.getLLMObsParentAgentSpanIdTagValue() != null) { + tagMap.put( + LLMOBS_PAGENT_SPAN_ID_TAG.forType(Encoding.DATADOG).toString(), + propagationTags + .getLLMObsParentAgentSpanIdTagValue() + .forType(Encoding.DATADOG) + .toString()); + } + if (propagationTags.getLLMObsParentAgentNameTagValue() != null) { + tagMap.put( + LLMOBS_PAGENT_NAME_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsParentAgentNameTagValue().forType(Encoding.DATADOG).toString()); + } if (propagationTags.getError() != null) { tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java index 0b5184d448a..a93fccc3bd0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java @@ -4,6 +4,10 @@ import static datadog.trace.core.propagation.PropagationTags.HeaderType.W3C; import static datadog.trace.core.propagation.ptags.PTagsCodec.DECISION_MAKER_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.KNUTH_SAMPLING_RATE_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_ML_APP_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_NAME_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_SPAN_ID_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_SESSION_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.ORG_PROPAGATION_MARKER_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_SOURCE_TAG; @@ -50,7 +54,7 @@ PTagsCodec getDecoderEncoder(@Nonnull HeaderType headerType) { @Override public final PropagationTags empty() { - return createValid(null, null, null, ProductTraceSource.UNSET, null); + return createValid(null, null, null, ProductTraceSource.UNSET, null, null); } @Override @@ -71,14 +75,16 @@ PropagationTags createValid( TagValue decisionMakerTagValue, TagValue traceIdTagValue, int productTraceSource, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { return new PTags( this, tagPairs, decisionMakerTagValue, traceIdTagValue, productTraceSource, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); } PropagationTags createInvalid(String error) { @@ -112,6 +118,11 @@ static class PTags extends PropagationTags { private volatile TagValue orgPropagationMarkerTagValue; + private volatile TagValue llmObsMlAppTagValue; + private volatile TagValue llmObsSessionIdTagValue; + private volatile TagValue llmObsParentAgentSpanIdTagValue; + private volatile TagValue llmObsParentAgentNameTagValue; + // Static cache for the most-recently-seen rate → TagValue. In steady state a service uses one // rate, so this eliminates the char[] + String allocation on every new PTags instance. // Writes are benign-racy: two threads computing the same rate produce equal TagValues. @@ -158,7 +169,8 @@ static class PTags extends PropagationTags { TagValue decisionMakerTagValue, TagValue traceIdTagValue, int traceSource, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { this( factory, tagPairs, @@ -168,7 +180,8 @@ static class PTags extends PropagationTags { PrioritySampling.UNSET, null, null, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); } PTags( @@ -180,7 +193,8 @@ static class PTags extends PropagationTags { int samplingPriority, CharSequence origin, CharSequence lastParentId, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { assert tagPairs == null || tagPairs.size() % 2 == 0; this.factory = factory; this.tagPairs = tagPairs; @@ -191,6 +205,11 @@ static class PTags extends PropagationTags { this.origin = origin; this.lastParentId = lastParentId; this.orgPropagationMarkerTagValue = orgPropagationMarkerTagValue; + LLMObsTagValues lov = llmObsTagValues != null ? llmObsTagValues : LLMObsTagValues.EMPTY; + this.llmObsMlAppTagValue = lov.mlApp; + this.llmObsSessionIdTagValue = lov.sessionId; + this.llmObsParentAgentSpanIdTagValue = lov.parentAgentSpanId; + this.llmObsParentAgentNameTagValue = lov.parentAgentName; if (traceIdTagValue != null) { CharSequence traceIdHighOrderBitsHex = traceIdTagValue.forType(TagElement.Encoding.DATADOG); this.traceIdHighOrderBits = @@ -212,6 +231,7 @@ static PTags withError(PTagsFactory factory, String error) { PrioritySampling.UNSET, null, null, + null, null); pTags.error = error; return pTags; @@ -377,6 +397,96 @@ TagValue getOrgPropagationMarkerTagValue() { return orgPropagationMarkerTagValue; } + @Override + public CharSequence getLLMObsMlApp() { + return llmObsMlAppTagValue; + } + + @Override + public void updateLLMObsMlApp(CharSequence mlApp) { + TagValue newValue = toTagValue(mlApp); + if (!Objects.equals(this.llmObsMlAppTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsMlAppTagValue = newValue; + } + } + + TagValue getLLMObsMlAppTagValue() { + return llmObsMlAppTagValue; + } + + @Override + public CharSequence getLLMObsSessionId() { + return llmObsSessionIdTagValue; + } + + @Override + public void updateLLMObsSessionId(CharSequence sessionId) { + TagValue newValue = toTagValue(sessionId); + if (!Objects.equals(this.llmObsSessionIdTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsSessionIdTagValue = newValue; + } + } + + TagValue getLLMObsSessionIdTagValue() { + return llmObsSessionIdTagValue; + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return llmObsParentAgentSpanIdTagValue; + } + + @Override + public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { + TagValue newValue = toTagValue(parentAgentSpanId); + if (!Objects.equals(this.llmObsParentAgentSpanIdTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsParentAgentSpanIdTagValue = newValue; + } + } + + TagValue getLLMObsParentAgentSpanIdTagValue() { + return llmObsParentAgentSpanIdTagValue; + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return llmObsParentAgentNameTagValue; + } + + @Override + public void updateLLMObsParentAgentName(CharSequence parentAgentName) { + TagValue newValue = toTagValue(parentAgentName); + if (!Objects.equals(this.llmObsParentAgentNameTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsParentAgentNameTagValue = newValue; + } + } + + TagValue getLLMObsParentAgentNameTagValue() { + return llmObsParentAgentNameTagValue; + } + + /** + * Wraps a non-empty value as a {@link TagValue}, or {@code null} if empty. No length capping is + * applied here — matching dd-trace-py, which writes these free-form values (ml_app, session_id, + * agent id/name) as-is and relies on the codecs' own overflow handling (dropping the whole + * {@code x-datadog-tags} header on the Datadog codec, or dropping individual overlong tags on + * the W3C codec) rather than a fixed per-field character limit. + */ + private static TagValue toTagValue(CharSequence value) { + if (value == null || value.length() == 0) { + return null; + } + return TagValue.from(value); + } + @Override public int getSamplingPriority() { return samplingPriority; @@ -512,6 +622,15 @@ int getXDatadogTagsSize() { size = PTagsCodec.calcXDatadogTagsSize( size, ORG_PROPAGATION_MARKER_TAG, getOrgPropagationMarkerTagValue()); + size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_ML_APP_TAG, llmObsMlAppTagValue); + size = + PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_SESSION_ID_TAG, llmObsSessionIdTagValue); + size = + PTagsCodec.calcXDatadogTagsSize( + size, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsParentAgentSpanIdTagValue); + size = + PTagsCodec.calcXDatadogTagsSize( + size, LLMOBS_PAGENT_NAME_TAG, llmObsParentAgentNameTagValue); int currentProductTraceSource = traceSource; if (currentProductTraceSource != ProductTraceSource.UNSET) { size = diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index c0018544188..0a6e18bd10a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -99,6 +99,10 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { int maxUnknownSize = 0; CharSequence lastParentId = null; TagValue orgPropagationMarkerTagValue = null; + TagValue llmObsMlAppTagValue = null; + TagValue llmObsSessionIdTagValue = null; + TagValue llmObsParentAgentSpanIdTagValue = null; + TagValue llmObsParentAgentNameTagValue = null; while (tagPos < ddMemberValueEnd) { tagPos = skipEmptyElements(value, tagPos, ddMemberValueEnd); if (tagPos >= ddMemberValueEnd) { @@ -168,6 +172,14 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { traceSource = ProductTraceSource.parseBitfieldHex(tagValue.toString()); } else if (tagKey.equals(ORG_PROPAGATION_MARKER_TAG)) { orgPropagationMarkerTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_ML_APP_TAG)) { + llmObsMlAppTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_SESSION_ID_TAG)) { + llmObsSessionIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_SPAN_ID_TAG)) { + llmObsParentAgentSpanIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { + llmObsParentAgentNameTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -201,7 +213,12 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { ddMemberValueEnd, maxUnknownSize, lastParentId, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + new LLMObsTagValues( + llmObsMlAppTagValue, + llmObsSessionIdTagValue, + llmObsParentAgentSpanIdTagValue, + llmObsParentAgentNameTagValue)); } @Override @@ -764,6 +781,7 @@ private static W3CPTags empty( ddMemberValueEnd, 0, null, + null, null); } @@ -799,7 +817,8 @@ public W3CPTags( int ddMemberValueEnd, int maxUnknownSize, CharSequence lastParentId, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { super( factory, tagPairs, @@ -809,7 +828,8 @@ public W3CPTags( samplingPriority, origin, lastParentId, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); this.tracestate = original; this.firstMemberStart = firstMemberStart; this.ddMemberStart = ddMemberStart; diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java index 1e768846b10..ed3a5bb68a4 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java @@ -19,6 +19,7 @@ private LLMObsContext() { } private static final ContextKey CONTEXT_KEY = ContextKey.named("llmobs_span"); + private static final ContextKey ML_APP_KEY = ContextKey.named("llmobs_ml_app"); private static final ContextKey SESSION_ID_KEY = ContextKey.named("llmobs_session_id"); private static final ContextKey AGENT_VERSION_KEY = ContextKey.named("llmobs_agent_version"); @@ -108,9 +109,38 @@ public static ContextScope attach( String samplingDecision, String parentAgentSpanId, String parentAgentName) { + return attach( + ctx, + null, + sessionId, + agentVersion, + sampleRate, + samplingDecision, + parentAgentSpanId, + parentAgentName); + } + + /** + * Attach an LLMObs span context, propagating ml_app alongside everything {@link + * #attach(AgentSpanContext, String, String, String, String, String, String)} carries. + * + *

ml_app is held here — rather than only as a span tag — so that distributed propagation can + * read the innermost active LLMObs span's ml_app when injecting, without needing a reference to + * the span itself. + */ + public static ContextScope attach( + AgentSpanContext ctx, + String mlApp, + String sessionId, + String agentVersion, + String sampleRate, + String samplingDecision, + String parentAgentSpanId, + String parentAgentName) { String decision = emptyToNull(samplingDecision); return Context.current() .with(CONTEXT_KEY, ctx) + .with(ML_APP_KEY, emptyToNull(mlApp)) .with(SESSION_ID_KEY, emptyToNull(sessionId)) .with(AGENT_VERSION_KEY, emptyToNull(agentVersion)) .with(SAMPLING_DECISION_KEY, decision) @@ -124,6 +154,11 @@ public static AgentSpanContext current() { return Context.current().get(CONTEXT_KEY); } + /** Return the ml_app of the innermost active LLMObs span, or null if none is active. */ + public static String currentMlApp() { + return Context.current().get(ML_APP_KEY); + } + /** * Return the session_id propagated from an enclosing LLMObs span, or null if no parent set one. */ diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java index 46a01b3f70f..0cf43469604 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java @@ -23,6 +23,11 @@ public final class AgentPropagation { // TODO DSM propagator should run after the other propagators as it stores the pathway context // TODO into the span context for now. Remove priority after the migration is complete. public static final Concern DSM_CONCERN = withPriority("data-stream-monitoring", 110); + // LLM Observability contributes no headers of its own: it stages the _dd.p.llmobs_* propagation + // tags onto the span context, which the tracing propagator then serializes into x-datadog-tags / + // tracestate. Composite injection runs in reverse priority order, so this must sort after + // TRACING_CONCERN to actually inject before it. + public static final Concern LLMOBS_CONCERN = withPriority("llm-observability", 115); private AgentPropagation() {} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 1dba9438168..77466ef92cd 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -55,6 +55,58 @@ default void mergePathwayContext(PathwayContext pathwayContext) {} default void setIntegrationName(CharSequence componentName) {} + /** + * Gets the LLM Observability {@code ml_app} propagated with this trace, or {@code null} if none + * is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsMlApp() { + return null; + } + + /** Sets the LLM Observability {@code ml_app} to propagate with this trace. No-op by default. */ + default void updateLLMObsMlApp(CharSequence mlApp) {} + + /** + * Gets the LLM Observability {@code session_id} propagated with this trace, or {@code null} if + * none is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsSessionId() { + return null; + } + + /** + * Sets the LLM Observability {@code session_id} to propagate with this trace. No-op by default. + */ + default void updateLLMObsSessionId(CharSequence sessionId) {} + + /** + * Gets the span id of the parent LLM Observability agent span propagated with this trace, or + * {@code null} if none is set or this context implementation doesn't have propagation-tags + * access. + */ + default CharSequence getLLMObsParentAgentSpanId() { + return null; + } + + /** + * Sets the parent LLM Observability agent span id to propagate with this trace. No-op by default. + */ + default void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) {} + + /** + * Gets the name of the parent LLM Observability agent span propagated with this trace, or {@code + * null} if none is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsParentAgentName() { + return null; + } + + /** + * Sets the parent LLM Observability agent span name to propagate with this trace. No-op by + * default. + */ + default void updateLLMObsParentAgentName(CharSequence parentAgentName) {} + /** * Gets whether the span context used is part of the local trace or from another service * From 4f08a5fae0cfdc5be62507d30f0bf468cb53ae0e Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Tue, 8 Sep 2026 10:26:56 -0400 Subject: [PATCH 2/5] Propagate LLM Observability parent span id across service boundaries Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/LLMObsContextPropagator.java | 4 ++++ .../trace/llmobs/domain/DDLLMObsSpan.java | 7 ++++++ .../llmobs/LLMObsContextPropagatorTest.java | 11 +++++++++ .../datadog/trace/core/DDSpanContext.java | 10 ++++++++ .../core/propagation/ExtractedContext.java | 5 ++++ .../core/propagation/PropagationTags.java | 9 ++++++++ .../propagation/ptags/DatadogPTagsCodec.java | 6 ++++- .../propagation/ptags/LLMObsTagValues.java | 16 +++++++++---- .../core/propagation/ptags/PTagsCodec.java | 9 ++++++++ .../core/propagation/ptags/PTagsFactory.java | 23 +++++++++++++++++++ .../core/propagation/ptags/W3CPTagsCodec.java | 6 ++++- .../instrumentation/api/AgentSpanContext.java | 11 +++++++++ 12 files changed, 110 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java index 92d6f3e0e26..c27ffc6b6a2 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java @@ -50,6 +50,10 @@ public void inject(Context context, C carrier, CarrierSetter setter) { spanContext.updateLLMObsSessionId(LLMObsContext.currentSessionId()); spanContext.updateLLMObsParentAgentSpanId(LLMObsContext.currentParentAgentSpanId()); spanContext.updateLLMObsParentAgentName(LLMObsContext.currentParentAgentName()); + // The innermost active LLMObs span becomes the downstream span's LLMObs parent, so the + // continued trace is a single tree rather than a second root per service. Mirrors + // dd-trace-py's _dd.p.llmobs_parent_id. + spanContext.updateLLMObsParentId(String.valueOf(llmObsContext.getSpanId())); } @Override diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 7dd69393a0f..ac693f9e79f 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -207,6 +207,13 @@ public DDLLMObsSpan( // // This also covers the trace-mismatch branch above: a stale context leaked from an unrelated // trace must not suppress attribution that legitimately arrived over the wire. + // Unlike dd-trace-py, a missing parent_id doesn't veto the rest: session and attribution + // are inherited independently, so a partially populated upstream still contributes what it + // did send. + String propagatedParentId = asString(span.spanContext().getLLMObsParentId()); + if (propagatedParentId != null) { + parentSpanID = propagatedParentId; + } if (sessionId == null || sessionId.isEmpty()) { sessionId = asString(span.spanContext().getLLMObsSessionId()); } diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java index cbde80af5ba..b65270761e3 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagatorTest.java @@ -37,6 +37,7 @@ class LLMObsContextPropagatorTest { private static final String SESSION_ID_TAG = "_dd.p.llmobs_sid"; private static final String PAGENT_SPAN_ID_TAG = "_dd.p.llmobs_pagent_span_id"; private static final String PAGENT_NAME_TAG = "_dd.p.llmobs_pagent_name"; + private static final String PARENT_ID_TAG = "_dd.p.llmobs_parent_id"; private static CoreTracer tracer; @@ -94,6 +95,8 @@ void stagesLlmObsTagsOnInjectionWithoutAnyManualPropagation() { () -> "pagent_span_id missing from " + tags); assertTrue( tags.contains(PAGENT_NAME_TAG + "=planner"), () -> "pagent_name missing from " + tags); + assertTrue( + tags.contains(PARENT_ID_TAG + "=" + agentSpanId), () -> "parent_id missing from " + tags); } @Test @@ -122,6 +125,9 @@ void stopsContributingTagsOnceTheLlmObsScopeIsClosed() { assertTrue( tags == null || !tags.contains(SESSION_ID_TAG), () -> "session_id leaked after scope close: " + tags); + assertTrue( + tags == null || !tags.contains(PARENT_ID_TAG), + () -> "parent_id leaked after scope close: " + tags); } /** @@ -164,6 +170,10 @@ void workerInheritsSessionAndAgentAttributionAcrossTheBoundary() { assertEquals("sess-42", LLMObsContext.currentSessionId()); assertEquals(producerAgentSpanId, LLMObsContext.currentParentAgentSpanId()); assertEquals("dispatcher", LLMObsContext.currentParentAgentName()); + // The worker's LLMObs span parents onto the producer's, rather than starting a second + // root — this is the value DDLLMObsSpan reads for its parent_id. + assertEquals( + producerAgentSpanId, String.valueOf(consumeSpan.spanContext().getLLMObsParentId())); } finally { workerTool.finish(); } @@ -189,6 +199,7 @@ void workerWithoutUpstreamLlmObsContextInheritsNothing() { try { assertNull(LLMObsContext.currentSessionId()); assertNull(LLMObsContext.currentParentAgentSpanId()); + assertNull(consumeSpan.spanContext().getLLMObsParentId()); } finally { workerTool.finish(); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 73b3e2b24e1..3732e73b547 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -1532,6 +1532,16 @@ public void updateLLMObsParentAgentName(CharSequence parentAgentName) { getPropagationTags().updateLLMObsParentAgentName(parentAgentName); } + @Override + public CharSequence getLLMObsParentId() { + return getPropagationTags().getLLMObsParentId(); + } + + @Override + public void updateLLMObsParentId(CharSequence parentId) { + getPropagationTags().updateLLMObsParentId(parentId); + } + /** TraceSegment Implementation */ @Override public void setTagTop(String key, Object value, boolean sanitize) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java index 52a40a94e4c..2e22251f285 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java @@ -137,6 +137,11 @@ public CharSequence getLLMObsParentAgentName() { return propagationTags.getLLMObsParentAgentName(); } + @Override + public CharSequence getLLMObsParentId() { + return propagationTags.getLLMObsParentId(); + } + @Override public String toString() { StringBuilder builder = new StringBuilder("ExtractedContext{"); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java index 47161ef1276..ba19faf486d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java @@ -205,6 +205,15 @@ public interface Factory { /** Sets the parent LLM Observability agent span name to propagate with this trace. */ public abstract void updateLLMObsParentAgentName(CharSequence parentAgentName); + /** + * Returns the span id of the parent LLM Observability span currently propagated with this trace, + * encoded as {@code _dd.p.llmobs_parent_id}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsParentId(); + + /** Sets the parent LLM Observability span id to propagate with this trace. */ + public abstract void updateLLMObsParentId(CharSequence parentId); + public HashMap createTagMap() { HashMap result = new HashMap<>(); fillTagMap(result); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index 907fab25e36..f1e3a5dcb94 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -68,6 +68,7 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { TagValue llmObsSessionIdTagValue = null; TagValue llmObsParentAgentSpanIdTagValue = null; TagValue llmObsParentAgentNameTagValue = null; + TagValue llmObsParentIdTagValue = null; while (tagPos < len) { int tagKeyEndsAt = validateCharsUntilSeparatorOrEnd( @@ -114,6 +115,8 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { llmObsParentAgentSpanIdTagValue = tagValue; } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { llmObsParentAgentNameTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PARENT_ID_TAG)) { + llmObsParentIdTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -136,7 +139,8 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { llmObsMlAppTagValue, llmObsSessionIdTagValue, llmObsParentAgentSpanIdTagValue, - llmObsParentAgentNameTagValue)); + llmObsParentAgentNameTagValue, + llmObsParentIdTagValue)); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java index 7d34fdab011..9551b71351c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java @@ -1,23 +1,29 @@ package datadog.trace.core.propagation.ptags; /** - * Bundles the four LLM Observability propagation tag values ({@code ml_app}, {@code session_id}, - * parent agent span id, parent agent name) extracted from an incoming header, so they can be - * threaded through {@link PTagsFactory.PTags} construction as a single parameter. + * Bundles the five LLM Observability propagation tag values ({@code ml_app}, {@code session_id}, + * parent agent span id, parent agent name, parent span id) extracted from an incoming header, so + * they can be threaded through {@link PTagsFactory.PTags} construction as a single parameter. */ final class LLMObsTagValues { - static final LLMObsTagValues EMPTY = new LLMObsTagValues(null, null, null, null); + static final LLMObsTagValues EMPTY = new LLMObsTagValues(null, null, null, null, null); final TagValue mlApp; final TagValue sessionId; final TagValue parentAgentSpanId; final TagValue parentAgentName; + final TagValue parentId; LLMObsTagValues( - TagValue mlApp, TagValue sessionId, TagValue parentAgentSpanId, TagValue parentAgentName) { + TagValue mlApp, + TagValue sessionId, + TagValue parentAgentSpanId, + TagValue parentAgentName, + TagValue parentId) { this.mlApp = mlApp; this.sessionId = sessionId; this.parentAgentSpanId = parentAgentSpanId; this.parentAgentName = parentAgentName; + this.parentId = parentId; } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index 99875e7f0a3..85a2e9f55c6 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -27,6 +27,7 @@ abstract class PTagsCodec { protected static final TagKey LLMOBS_SESSION_ID_TAG = TagKey.from("llmobs_sid"); protected static final TagKey LLMOBS_PAGENT_SPAN_ID_TAG = TagKey.from("llmobs_pagent_span_id"); protected static final TagKey LLMOBS_PAGENT_NAME_TAG = TagKey.from("llmobs_pagent_name"); + protected static final TagKey LLMOBS_PARENT_ID_TAG = TagKey.from("llmobs_parent_id"); static String headerValue(PTagsCodec codec, PTags ptags) { return headerValue(codec, ptags, null); @@ -85,6 +86,9 @@ static String headerValue(PTagsCodec codec, PTags ptags, CharSequence lastParent codec.appendTag( sb, LLMOBS_PAGENT_NAME_TAG, ptags.getLLMObsParentAgentNameTagValue(), size); } + if (ptags.getLLMObsParentIdTagValue() != null) { + size = codec.appendTag(sb, LLMOBS_PARENT_ID_TAG, ptags.getLLMObsParentIdTagValue(), size); + } Iterator it = ptags.getTagPairs().iterator(); while (it.hasNext() && !codec.isTooLarge(sb, size)) { TagElement tagKey = it.next(); @@ -180,6 +184,11 @@ static void fillTagMap(PTags propagationTags, Map tagMap) { LLMOBS_PAGENT_NAME_TAG.forType(Encoding.DATADOG).toString(), propagationTags.getLLMObsParentAgentNameTagValue().forType(Encoding.DATADOG).toString()); } + if (propagationTags.getLLMObsParentIdTagValue() != null) { + tagMap.put( + LLMOBS_PARENT_ID_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsParentIdTagValue().forType(Encoding.DATADOG).toString()); + } if (propagationTags.getError() != null) { tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java index a93fccc3bd0..775a697e87f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java @@ -7,6 +7,7 @@ import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_ML_APP_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_NAME_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_SPAN_ID_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PARENT_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_SESSION_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.ORG_PROPAGATION_MARKER_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_ID_TAG; @@ -122,6 +123,7 @@ static class PTags extends PropagationTags { private volatile TagValue llmObsSessionIdTagValue; private volatile TagValue llmObsParentAgentSpanIdTagValue; private volatile TagValue llmObsParentAgentNameTagValue; + private volatile TagValue llmObsParentIdTagValue; // Static cache for the most-recently-seen rate → TagValue. In steady state a service uses one // rate, so this eliminates the char[] + String allocation on every new PTags instance. @@ -210,6 +212,7 @@ static class PTags extends PropagationTags { this.llmObsSessionIdTagValue = lov.sessionId; this.llmObsParentAgentSpanIdTagValue = lov.parentAgentSpanId; this.llmObsParentAgentNameTagValue = lov.parentAgentName; + this.llmObsParentIdTagValue = lov.parentId; if (traceIdTagValue != null) { CharSequence traceIdHighOrderBitsHex = traceIdTagValue.forType(TagElement.Encoding.DATADOG); this.traceIdHighOrderBits = @@ -473,6 +476,25 @@ TagValue getLLMObsParentAgentNameTagValue() { return llmObsParentAgentNameTagValue; } + @Override + public CharSequence getLLMObsParentId() { + return llmObsParentIdTagValue; + } + + @Override + public void updateLLMObsParentId(CharSequence parentId) { + TagValue newValue = toTagValue(parentId); + if (!Objects.equals(this.llmObsParentIdTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsParentIdTagValue = newValue; + } + } + + TagValue getLLMObsParentIdTagValue() { + return llmObsParentIdTagValue; + } + /** * Wraps a non-empty value as a {@link TagValue}, or {@code null} if empty. No length capping is * applied here — matching dd-trace-py, which writes these free-form values (ml_app, session_id, @@ -631,6 +653,7 @@ int getXDatadogTagsSize() { size = PTagsCodec.calcXDatadogTagsSize( size, LLMOBS_PAGENT_NAME_TAG, llmObsParentAgentNameTagValue); + size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_PARENT_ID_TAG, llmObsParentIdTagValue); int currentProductTraceSource = traceSource; if (currentProductTraceSource != ProductTraceSource.UNSET) { size = diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index 0a6e18bd10a..fc34fe043c0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -103,6 +103,7 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { TagValue llmObsSessionIdTagValue = null; TagValue llmObsParentAgentSpanIdTagValue = null; TagValue llmObsParentAgentNameTagValue = null; + TagValue llmObsParentIdTagValue = null; while (tagPos < ddMemberValueEnd) { tagPos = skipEmptyElements(value, tagPos, ddMemberValueEnd); if (tagPos >= ddMemberValueEnd) { @@ -180,6 +181,8 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { llmObsParentAgentSpanIdTagValue = tagValue; } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { llmObsParentAgentNameTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PARENT_ID_TAG)) { + llmObsParentIdTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -218,7 +221,8 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { llmObsMlAppTagValue, llmObsSessionIdTagValue, llmObsParentAgentSpanIdTagValue, - llmObsParentAgentNameTagValue)); + llmObsParentAgentNameTagValue, + llmObsParentIdTagValue)); } @Override diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 77466ef92cd..4273f75948b 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -107,6 +107,17 @@ default CharSequence getLLMObsParentAgentName() { */ default void updateLLMObsParentAgentName(CharSequence parentAgentName) {} + /** + * Gets the span id of the parent LLM Observability span propagated with this trace, or {@code + * null} if none is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsParentId() { + return null; + } + + /** Sets the parent LLM Observability span id to propagate with this trace. No-op by default. */ + default void updateLLMObsParentId(CharSequence parentId) {} + /** * Gets whether the span context used is part of the local trace or from another service * From 77546ab8f7bea41d460a69535857a12572a0ec24 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Tue, 8 Sep 2026 10:41:45 -0400 Subject: [PATCH 3/5] Extract propagation tags from the _datadog message attribute Co-Authored-By: Claude Opus 5 --- .../messaging/DatadogAttributeParser.java | 4 + .../messaging/DatadogAttributeParserTest.java | 119 ++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParserTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParser.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParser.java index 4a901f9b370..94b5d836064 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParser.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParser.java @@ -24,6 +24,10 @@ public static void forEachProperty(AgentPropagation.KeyClassifier classifier, St if (acceptJsonProperty(classifier, json, "x-datadog-trace-id")) { acceptJsonProperty(classifier, json, "x-datadog-parent-id"); acceptJsonProperty(classifier, json, "x-datadog-sampling-priority"); + // Propagation tags travel in x-datadog-tags. Without this the whole _dd.p.* set is + // silently dropped at a messaging boundary — including _dd.p.tid, which truncates a + // 128-bit trace id to 64 bits downstream, and the _dd.p.llmobs_* attribution tags. + acceptJsonProperty(classifier, json, "x-datadog-tags"); } if (Config.get().isDataStreamsEnabled()) { acceptJsonProperty(classifier, json, "dd-pathway-ctx-base64"); diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParserTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParserTest.java new file mode 100644 index 00000000000..446a06a0c29 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/messaging/DatadogAttributeParserTest.java @@ -0,0 +1,119 @@ +package datadog.trace.bootstrap.instrumentation.messaging; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Covers the {@code _datadog} message attribute parser shared by the AWS messaging instrumentations + * (SQS, SNS, EventBridge, Step Functions). + */ +class DatadogAttributeParserTest { + + /** What an injected {@code _datadog} attribute looks like on the wire. */ + private static final String FULL_CONTEXT = + "{\"x-datadog-trace-id\":\"1234567890\"," + + "\"x-datadog-parent-id\":\"9876543210\"," + + "\"x-datadog-sampling-priority\":\"1\"," + + "\"x-datadog-tags\":\"_dd.p.dm=-1,_dd.p.tid=6aa01c5400000000\"," + + "\"traceparent\":\"00-6aa01c5400000000499602d2-000000024cb016ea-01\"}"; + + private static Map parse(String json) { + Map collected = new LinkedHashMap<>(); + DatadogAttributeParser.forEachProperty( + (key, value) -> { + collected.put(key, value); + return true; + }, + json); + return collected; + } + + @Test + void extractsTraceContextAndPropagationTags() { + Map collected = parse(FULL_CONTEXT); + + assertEquals("1234567890", collected.get("x-datadog-trace-id")); + assertEquals("9876543210", collected.get("x-datadog-parent-id")); + assertEquals("1", collected.get("x-datadog-sampling-priority")); + // Without x-datadog-tags the whole _dd.p.* set is dropped at the messaging boundary: the + // 64-bit trace id still joins, but _dd.p.tid is lost so the two services disagree about the + // full 128-bit id, and _dd.p.dm is lost with it. + assertEquals("_dd.p.dm=-1,_dd.p.tid=6aa01c5400000000", collected.get("x-datadog-tags")); + } + + @Test + void extractsPropagationTagsFromByteBufferCarrier() { + Map collected = new LinkedHashMap<>(); + DatadogAttributeParser.forEachProperty( + (key, value) -> { + collected.put(key, value); + return true; + }, + ByteBuffer.wrap(FULL_CONTEXT.getBytes(UTF_8))); + + assertEquals("_dd.p.dm=-1,_dd.p.tid=6aa01c5400000000", collected.get("x-datadog-tags")); + } + + @Test + void extractsPropagationTagsFromBase64ByteBufferCarrier() { + Map collected = new LinkedHashMap<>(); + DatadogAttributeParser.forEachProperty( + (key, value) -> { + collected.put(key, value); + return true; + }, + ByteBuffer.wrap(Base64.getEncoder().encode(FULL_CONTEXT.getBytes(UTF_8)))); + + assertEquals("_dd.p.dm=-1,_dd.p.tid=6aa01c5400000000", collected.get("x-datadog-tags")); + } + + @Test + void carriesLlmObsPropagationTags() { + Map collected = + parse( + "{\"x-datadog-trace-id\":\"1234567890\"," + + "\"x-datadog-parent-id\":\"9876543210\"," + + "\"x-datadog-tags\":\"_dd.p.llmobs_ml_app=my-app,_dd.p.llmobs_sid=sess-1," + + "_dd.p.llmobs_parent_id=42\"}"); + + String tags = collected.get("x-datadog-tags"); + assertTrue(tags.contains("_dd.p.llmobs_ml_app=my-app"), tags); + assertTrue(tags.contains("_dd.p.llmobs_sid=sess-1"), tags); + assertTrue(tags.contains("_dd.p.llmobs_parent_id=42"), tags); + } + + @Test + void extractsNothingWithoutATraceId() { + // Propagation tags on their own describe no trace, so they are not surfaced. + Map collected = + parse("{\"x-datadog-tags\":\"_dd.p.dm=-1\",\"x-datadog-parent-id\":\"9876543210\"}"); + + assertTrue(collected.isEmpty(), () -> "expected nothing extracted, got " + collected); + } + + @Test + void toleratesAMissingTagsProperty() { + Map collected = + parse( + "{\"x-datadog-trace-id\":\"1234567890\",\"x-datadog-parent-id\":\"9876543210\"," + + "\"x-datadog-sampling-priority\":\"1\"}"); + + assertEquals("1234567890", collected.get("x-datadog-trace-id")); + assertNull(collected.get("x-datadog-tags")); + } + + @Test + void toleratesMalformedJson() { + assertTrue(parse("not json at all").isEmpty()); + assertTrue(parse("{\"x-datadog-trace-id\":").isEmpty()); + assertTrue(parse(null).isEmpty()); + } +} From f26dec063ba4b8c8b4da4fd3ba3ed261e7efbd6d Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Tue, 8 Sep 2026 14:12:48 -0400 Subject: [PATCH 4/5] Update LLM Observability propagation tags as one atomic bundle Replace the five per-tag setters with a single updateLLMObsContext across PropagationTags, AgentSpanContext, DDSpanContext and the injecting call site, and hold the values in PTags as one volatile LLMObsTagValues instead of five volatile fields. A concurrent reader can no longer serialize a header mixing values from two contexts, and getXDatadogTagsSize can no longer size a combination that never existed -- which matters because that total gates whether x-datadog-tags is emitted at all. Also make LLMObsTagValues non-null throughout (EMPTY plus an of() factory that reuses it, so the common no-LLMObs request doesn't allocate), and add a private clearCachedHeaders() for the 11 sites that invalidate both encodings, leaving the three deliberate single-encoding calls visibly deliberate. Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/LLMObsContextPropagator.java | 11 +- .../datadog/trace/core/DDSpanContext.java | 30 +--- .../core/propagation/PropagationTags.java | 25 ++- .../propagation/ptags/DatadogPTagsCodec.java | 2 +- .../propagation/ptags/LLMObsTagValues.java | 40 ++++- .../core/propagation/ptags/PTagsCodec.java | 69 +++---- .../core/propagation/ptags/PTagsFactory.java | 169 +++++++----------- .../core/propagation/ptags/W3CPTagsCodec.java | 4 +- .../instrumentation/api/AgentSpanContext.java | 33 ++-- 9 files changed, 171 insertions(+), 212 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java index c27ffc6b6a2..056358b8f90 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java @@ -46,14 +46,15 @@ public void inject(Context context, C carrier, CarrierSetter setter) { return; } - spanContext.updateLLMObsMlApp(LLMObsContext.currentMlApp()); - spanContext.updateLLMObsSessionId(LLMObsContext.currentSessionId()); - spanContext.updateLLMObsParentAgentSpanId(LLMObsContext.currentParentAgentSpanId()); - spanContext.updateLLMObsParentAgentName(LLMObsContext.currentParentAgentName()); // The innermost active LLMObs span becomes the downstream span's LLMObs parent, so the // continued trace is a single tree rather than a second root per service. Mirrors // dd-trace-py's _dd.p.llmobs_parent_id. - spanContext.updateLLMObsParentId(String.valueOf(llmObsContext.getSpanId())); + spanContext.updateLLMObsContext( + LLMObsContext.currentMlApp(), + LLMObsContext.currentSessionId(), + LLMObsContext.currentParentAgentSpanId(), + LLMObsContext.currentParentAgentName(), + String.valueOf(llmObsContext.getSpanId())); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 3732e73b547..44d9862fba4 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -1498,8 +1498,14 @@ public CharSequence getLLMObsMlApp() { } @Override - public void updateLLMObsMlApp(CharSequence mlApp) { - getPropagationTags().updateLLMObsMlApp(mlApp); + public void updateLLMObsContext( + CharSequence mlApp, + CharSequence sessionId, + CharSequence parentAgentSpanId, + CharSequence parentAgentName, + CharSequence parentId) { + getPropagationTags() + .updateLLMObsContext(mlApp, sessionId, parentAgentSpanId, parentAgentName, parentId); } @Override @@ -1507,41 +1513,21 @@ public CharSequence getLLMObsSessionId() { return getPropagationTags().getLLMObsSessionId(); } - @Override - public void updateLLMObsSessionId(CharSequence sessionId) { - getPropagationTags().updateLLMObsSessionId(sessionId); - } - @Override public CharSequence getLLMObsParentAgentSpanId() { return getPropagationTags().getLLMObsParentAgentSpanId(); } - @Override - public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { - getPropagationTags().updateLLMObsParentAgentSpanId(parentAgentSpanId); - } - @Override public CharSequence getLLMObsParentAgentName() { return getPropagationTags().getLLMObsParentAgentName(); } - @Override - public void updateLLMObsParentAgentName(CharSequence parentAgentName) { - getPropagationTags().updateLLMObsParentAgentName(parentAgentName); - } - @Override public CharSequence getLLMObsParentId() { return getPropagationTags().getLLMObsParentId(); } - @Override - public void updateLLMObsParentId(CharSequence parentId) { - getPropagationTags().updateLLMObsParentId(parentId); - } - /** TraceSegment Implementation */ @Override public void setTagTop(String key, Object value, boolean sanitize) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java index ba19faf486d..39d2ceff1e0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java @@ -175,8 +175,17 @@ public interface Factory { */ public abstract CharSequence getLLMObsMlApp(); - /** Sets the LLM Observability {@code ml_app} to propagate with this trace. */ - public abstract void updateLLMObsMlApp(CharSequence mlApp); + /** + * Sets the whole LLM Observability tag set to propagate with this trace, replacing any set + * previously staged. Taken together rather than one tag at a time so the update is atomic: a + * concurrent reader never serializes a header mixing values from two different contexts. + */ + public abstract void updateLLMObsContext( + CharSequence mlApp, + CharSequence sessionId, + CharSequence parentAgentSpanId, + CharSequence parentAgentName, + CharSequence parentId); /** * Returns the LLM Observability {@code session_id} currently propagated with this trace, encoded @@ -184,36 +193,24 @@ public interface Factory { */ public abstract CharSequence getLLMObsSessionId(); - /** Sets the LLM Observability {@code session_id} to propagate with this trace. */ - public abstract void updateLLMObsSessionId(CharSequence sessionId); - /** * Returns the span id of the parent LLM Observability agent span currently propagated with this * trace, encoded as {@code _dd.p.llmobs_pagent_span_id}. Returns {@code null} if none is set. */ public abstract CharSequence getLLMObsParentAgentSpanId(); - /** Sets the parent LLM Observability agent span id to propagate with this trace. */ - public abstract void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId); - /** * Returns the name of the parent LLM Observability agent span currently propagated with this * trace, encoded as {@code _dd.p.llmobs_pagent_name}. Returns {@code null} if none is set. */ public abstract CharSequence getLLMObsParentAgentName(); - /** Sets the parent LLM Observability agent span name to propagate with this trace. */ - public abstract void updateLLMObsParentAgentName(CharSequence parentAgentName); - /** * Returns the span id of the parent LLM Observability span currently propagated with this trace, * encoded as {@code _dd.p.llmobs_parent_id}. Returns {@code null} if none is set. */ public abstract CharSequence getLLMObsParentId(); - /** Sets the parent LLM Observability span id to propagate with this trace. */ - public abstract void updateLLMObsParentId(CharSequence parentId); - public HashMap createTagMap() { HashMap result = new HashMap<>(); fillTagMap(result); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index f1e3a5dcb94..e8aa3a776d1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -135,7 +135,7 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { traceIdTagValue, traceSource, orgPropagationMarkerTagValue, - new LLMObsTagValues( + LLMObsTagValues.of( llmObsMlAppTagValue, llmObsSessionIdTagValue, llmObsParentAgentSpanIdTagValue, diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java index 9551b71351c..f3dabe3dae2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java @@ -1,9 +1,16 @@ package datadog.trace.core.propagation.ptags; +import java.util.Objects; + /** * Bundles the five LLM Observability propagation tag values ({@code ml_app}, {@code session_id}, * parent agent span id, parent agent name, parent span id) extracted from an incoming header, so * they can be threaded through {@link PTagsFactory.PTags} construction as a single parameter. + * + *

Never {@code null}: use {@link #EMPTY} to say "no LLM Observability tags", and obtain + * instances through {@link #of} so that the common case — an incoming request carrying none of + * these tags, which is every request in a service not using LLM Observability — reuses {@code + * EMPTY} rather than allocating. */ final class LLMObsTagValues { static final LLMObsTagValues EMPTY = new LLMObsTagValues(null, null, null, null, null); @@ -14,7 +21,24 @@ final class LLMObsTagValues { final TagValue parentAgentName; final TagValue parentId; - LLMObsTagValues( + /** Returns {@link #EMPTY} when every value is {@code null}, otherwise a new bundle. */ + static LLMObsTagValues of( + TagValue mlApp, + TagValue sessionId, + TagValue parentAgentSpanId, + TagValue parentAgentName, + TagValue parentId) { + if (mlApp == null + && sessionId == null + && parentAgentSpanId == null + && parentAgentName == null + && parentId == null) { + return EMPTY; + } + return new LLMObsTagValues(mlApp, sessionId, parentAgentSpanId, parentAgentName, parentId); + } + + private LLMObsTagValues( TagValue mlApp, TagValue sessionId, TagValue parentAgentSpanId, @@ -26,4 +50,18 @@ final class LLMObsTagValues { this.parentAgentName = parentAgentName; this.parentId = parentId; } + + /** + * Whether {@code other} carries the same five values. Used to skip cache invalidation when an + * injection re-stages tags a span already has; not {@code equals} because these are never used as + * map keys and identity equality is the useful default elsewhere in this package. + */ + boolean sameAs(LLMObsTagValues other) { + return this == other + || (Objects.equals(mlApp, other.mlApp) + && Objects.equals(sessionId, other.sessionId) + && Objects.equals(parentAgentSpanId, other.parentAgentSpanId) + && Objects.equals(parentAgentName, other.parentAgentName) + && Objects.equals(parentId, other.parentId)); + } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index 85a2e9f55c6..9869d42f635 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -70,24 +70,22 @@ static String headerValue(PTagsCodec codec, PTags ptags, CharSequence lastParent codec.appendTag( sb, ORG_PROPAGATION_MARKER_TAG, ptags.getOrgPropagationMarkerTagValue(), size); } - if (ptags.getLLMObsMlAppTagValue() != null) { - size = codec.appendTag(sb, LLMOBS_ML_APP_TAG, ptags.getLLMObsMlAppTagValue(), size); + // One snapshot, so a concurrent injection can't have us encode a mix of old and new values. + LLMObsTagValues llmObsTags = ptags.getLLMObsTagValues(); + if (llmObsTags.mlApp != null) { + size = codec.appendTag(sb, LLMOBS_ML_APP_TAG, llmObsTags.mlApp, size); } - if (ptags.getLLMObsSessionIdTagValue() != null) { - size = codec.appendTag(sb, LLMOBS_SESSION_ID_TAG, ptags.getLLMObsSessionIdTagValue(), size); + if (llmObsTags.sessionId != null) { + size = codec.appendTag(sb, LLMOBS_SESSION_ID_TAG, llmObsTags.sessionId, size); } - if (ptags.getLLMObsParentAgentSpanIdTagValue() != null) { - size = - codec.appendTag( - sb, LLMOBS_PAGENT_SPAN_ID_TAG, ptags.getLLMObsParentAgentSpanIdTagValue(), size); + if (llmObsTags.parentAgentSpanId != null) { + size = codec.appendTag(sb, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsTags.parentAgentSpanId, size); } - if (ptags.getLLMObsParentAgentNameTagValue() != null) { - size = - codec.appendTag( - sb, LLMOBS_PAGENT_NAME_TAG, ptags.getLLMObsParentAgentNameTagValue(), size); + if (llmObsTags.parentAgentName != null) { + size = codec.appendTag(sb, LLMOBS_PAGENT_NAME_TAG, llmObsTags.parentAgentName, size); } - if (ptags.getLLMObsParentIdTagValue() != null) { - size = codec.appendTag(sb, LLMOBS_PARENT_ID_TAG, ptags.getLLMObsParentIdTagValue(), size); + if (llmObsTags.parentId != null) { + size = codec.appendTag(sb, LLMOBS_PARENT_ID_TAG, llmObsTags.parentId, size); } Iterator it = ptags.getTagPairs().iterator(); while (it.hasNext() && !codec.isTooLarge(sb, size)) { @@ -161,39 +159,26 @@ static void fillTagMap(PTags propagationTags, Map tagMap) { .forType(Encoding.DATADOG) .toString()); } - if (propagationTags.getLLMObsMlAppTagValue() != null) { - tagMap.put( - LLMOBS_ML_APP_TAG.forType(Encoding.DATADOG).toString(), - propagationTags.getLLMObsMlAppTagValue().forType(Encoding.DATADOG).toString()); - } - if (propagationTags.getLLMObsSessionIdTagValue() != null) { - tagMap.put( - LLMOBS_SESSION_ID_TAG.forType(Encoding.DATADOG).toString(), - propagationTags.getLLMObsSessionIdTagValue().forType(Encoding.DATADOG).toString()); - } - if (propagationTags.getLLMObsParentAgentSpanIdTagValue() != null) { - tagMap.put( - LLMOBS_PAGENT_SPAN_ID_TAG.forType(Encoding.DATADOG).toString(), - propagationTags - .getLLMObsParentAgentSpanIdTagValue() - .forType(Encoding.DATADOG) - .toString()); - } - if (propagationTags.getLLMObsParentAgentNameTagValue() != null) { - tagMap.put( - LLMOBS_PAGENT_NAME_TAG.forType(Encoding.DATADOG).toString(), - propagationTags.getLLMObsParentAgentNameTagValue().forType(Encoding.DATADOG).toString()); - } - if (propagationTags.getLLMObsParentIdTagValue() != null) { - tagMap.put( - LLMOBS_PARENT_ID_TAG.forType(Encoding.DATADOG).toString(), - propagationTags.getLLMObsParentIdTagValue().forType(Encoding.DATADOG).toString()); - } + LLMObsTagValues llmObsTags = propagationTags.getLLMObsTagValues(); + putLLMObsTag(tagMap, LLMOBS_ML_APP_TAG, llmObsTags.mlApp); + putLLMObsTag(tagMap, LLMOBS_SESSION_ID_TAG, llmObsTags.sessionId); + putLLMObsTag(tagMap, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsTags.parentAgentSpanId); + putLLMObsTag(tagMap, LLMOBS_PAGENT_NAME_TAG, llmObsTags.parentAgentName); + putLLMObsTag(tagMap, LLMOBS_PARENT_ID_TAG, llmObsTags.parentId); if (propagationTags.getError() != null) { tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); } } + /** Adds one LLM Observability tag to the span's tag map, skipping it when unset. */ + private static void putLLMObsTag(Map tagMap, TagKey tagKey, TagValue tagValue) { + if (tagValue != null) { + tagMap.put( + tagKey.forType(Encoding.DATADOG).toString(), + tagValue.forType(Encoding.DATADOG).toString()); + } + } + static int calcXDatadogTagsSize(List tagPairs) { int size = 0; int pl = Encoding.DATADOG.getPrefixLength(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java index 775a697e87f..2e10b18a532 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java @@ -55,7 +55,7 @@ PTagsCodec getDecoderEncoder(@Nonnull HeaderType headerType) { @Override public final PropagationTags empty() { - return createValid(null, null, null, ProductTraceSource.UNSET, null, null); + return createValid(null, null, null, ProductTraceSource.UNSET, null, LLMObsTagValues.EMPTY); } @Override @@ -77,7 +77,7 @@ PropagationTags createValid( TagValue traceIdTagValue, int productTraceSource, TagValue orgPropagationMarkerTagValue, - LLMObsTagValues llmObsTagValues) { + @Nonnull LLMObsTagValues llmObsTagValues) { return new PTags( this, tagPairs, @@ -119,11 +119,13 @@ static class PTags extends PropagationTags { private volatile TagValue orgPropagationMarkerTagValue; - private volatile TagValue llmObsMlAppTagValue; - private volatile TagValue llmObsSessionIdTagValue; - private volatile TagValue llmObsParentAgentSpanIdTagValue; - private volatile TagValue llmObsParentAgentNameTagValue; - private volatile TagValue llmObsParentIdTagValue; + /** + * The LLM Observability propagation tags, held as one immutable bundle rather than five fields + * so that an update is a single reference swap. Readers therefore always observe a tag set that + * actually existed, instead of a mix of values from before and after an injection. Never {@code + * null} — {@link LLMObsTagValues#EMPTY} means "none". + */ + private volatile LLMObsTagValues llmObsTags = LLMObsTagValues.EMPTY; // Static cache for the most-recently-seen rate → TagValue. In steady state a service uses one // rate, so this eliminates the char[] + String allocation on every new PTags instance. @@ -172,7 +174,7 @@ static class PTags extends PropagationTags { TagValue traceIdTagValue, int traceSource, TagValue orgPropagationMarkerTagValue, - LLMObsTagValues llmObsTagValues) { + @Nonnull LLMObsTagValues llmObsTagValues) { this( factory, tagPairs, @@ -196,7 +198,7 @@ static class PTags extends PropagationTags { CharSequence origin, CharSequence lastParentId, TagValue orgPropagationMarkerTagValue, - LLMObsTagValues llmObsTagValues) { + @Nonnull LLMObsTagValues llmObsTagValues) { assert tagPairs == null || tagPairs.size() % 2 == 0; this.factory = factory; this.tagPairs = tagPairs; @@ -207,12 +209,7 @@ static class PTags extends PropagationTags { this.origin = origin; this.lastParentId = lastParentId; this.orgPropagationMarkerTagValue = orgPropagationMarkerTagValue; - LLMObsTagValues lov = llmObsTagValues != null ? llmObsTagValues : LLMObsTagValues.EMPTY; - this.llmObsMlAppTagValue = lov.mlApp; - this.llmObsSessionIdTagValue = lov.sessionId; - this.llmObsParentAgentSpanIdTagValue = lov.parentAgentSpanId; - this.llmObsParentAgentNameTagValue = lov.parentAgentName; - this.llmObsParentIdTagValue = lov.parentId; + this.llmObsTags = llmObsTagValues; if (traceIdTagValue != null) { CharSequence traceIdHighOrderBitsHex = traceIdTagValue.forType(TagElement.Encoding.DATADOG); this.traceIdHighOrderBits = @@ -235,7 +232,7 @@ static PTags withError(PTagsFactory factory, String error) { null, null, null, - null); + LLMObsTagValues.EMPTY); pTags.error = error; return pTags; } @@ -271,8 +268,7 @@ private void doUpdateTraceSamplingPriority(int samplingPriority, int samplingMec TagValue newDM = TagValue.from("-" + samplingMechanism); if (!newDM.equals(decisionMakerTagValue)) { // This should invalidate any cached w3c and datadog header - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); } decisionMakerTagValue = newDM; } @@ -280,8 +276,7 @@ private void doUpdateTraceSamplingPriority(int samplingPriority, int samplingMec // Drop the decision maker tag if (decisionMakerTagValue != null) { // This should invalidate any cached w3c and datadog header - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); } decisionMakerTagValue = null; } @@ -298,8 +293,7 @@ public void addTraceSource(final int product) { } // Invalidate cached headers (atomic context ensures correctness) - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); // Set the bit for the given product return ProductTraceSource.updateProduct(currentValue, product); @@ -324,8 +318,7 @@ public String getDebugPropagation() { @Override public void updateKnuthSamplingRate(double rate) { if (Double.compare(knuthSamplingRate, rate) != 0) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); knuthSamplingRate = rate; if (Double.isNaN(rate)) { knuthSamplingRateTagValue = null; @@ -390,8 +383,7 @@ public CharSequence getOrgPropagationMarker() { public void updateOrgPropagationMarker(CharSequence opm) { TagValue newValue = opm == null ? null : TagValue.from(opm); if (!Objects.equals(this.orgPropagationMarkerTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); this.orgPropagationMarkerTagValue = newValue; } } @@ -401,98 +393,53 @@ TagValue getOrgPropagationMarkerTagValue() { } @Override - public CharSequence getLLMObsMlApp() { - return llmObsMlAppTagValue; - } - - @Override - public void updateLLMObsMlApp(CharSequence mlApp) { - TagValue newValue = toTagValue(mlApp); - if (!Objects.equals(this.llmObsMlAppTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); - this.llmObsMlAppTagValue = newValue; + public void updateLLMObsContext( + CharSequence mlApp, + CharSequence sessionId, + CharSequence parentAgentSpanId, + CharSequence parentAgentName, + CharSequence parentId) { + LLMObsTagValues updated = + LLMObsTagValues.of( + toTagValue(mlApp), + toTagValue(sessionId), + toTagValue(parentAgentSpanId), + toTagValue(parentAgentName), + toTagValue(parentId)); + // Re-injecting the same context onto the same span is the common case; don't invalidate. + if (!updated.sameAs(llmObsTags)) { + clearCachedHeaders(); + llmObsTags = updated; } } - TagValue getLLMObsMlAppTagValue() { - return llmObsMlAppTagValue; - } - @Override - public CharSequence getLLMObsSessionId() { - return llmObsSessionIdTagValue; + public CharSequence getLLMObsMlApp() { + return llmObsTags.mlApp; } @Override - public void updateLLMObsSessionId(CharSequence sessionId) { - TagValue newValue = toTagValue(sessionId); - if (!Objects.equals(this.llmObsSessionIdTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); - this.llmObsSessionIdTagValue = newValue; - } - } - - TagValue getLLMObsSessionIdTagValue() { - return llmObsSessionIdTagValue; + public CharSequence getLLMObsSessionId() { + return llmObsTags.sessionId; } @Override public CharSequence getLLMObsParentAgentSpanId() { - return llmObsParentAgentSpanIdTagValue; - } - - @Override - public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { - TagValue newValue = toTagValue(parentAgentSpanId); - if (!Objects.equals(this.llmObsParentAgentSpanIdTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); - this.llmObsParentAgentSpanIdTagValue = newValue; - } - } - - TagValue getLLMObsParentAgentSpanIdTagValue() { - return llmObsParentAgentSpanIdTagValue; + return llmObsTags.parentAgentSpanId; } @Override public CharSequence getLLMObsParentAgentName() { - return llmObsParentAgentNameTagValue; - } - - @Override - public void updateLLMObsParentAgentName(CharSequence parentAgentName) { - TagValue newValue = toTagValue(parentAgentName); - if (!Objects.equals(this.llmObsParentAgentNameTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); - this.llmObsParentAgentNameTagValue = newValue; - } - } - - TagValue getLLMObsParentAgentNameTagValue() { - return llmObsParentAgentNameTagValue; + return llmObsTags.parentAgentName; } @Override public CharSequence getLLMObsParentId() { - return llmObsParentIdTagValue; + return llmObsTags.parentId; } - @Override - public void updateLLMObsParentId(CharSequence parentId) { - TagValue newValue = toTagValue(parentId); - if (!Objects.equals(this.llmObsParentIdTagValue, newValue)) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); - this.llmObsParentIdTagValue = newValue; - } - } - - TagValue getLLMObsParentIdTagValue() { - return llmObsParentIdTagValue; + LLMObsTagValues getLLMObsTagValues() { + return llmObsTags; } /** @@ -605,6 +552,16 @@ private void setCachedHeader(HeaderType headerType, String header) { cache[headerType.ordinal()] = header; } + /** + * Invalidate every encoding's cached header, and the memoized x-datadog-tags size with them. + * Use this whenever a change affects both wire formats; the single-encoding {@link + * #clearCachedHeader} calls that remain are deliberate. + */ + private void clearCachedHeaders() { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + } + private void clearCachedHeader(HeaderType headerType) { if (headerType == DATADOG) { invalidateXDatadogTagsSize(); @@ -644,16 +601,21 @@ int getXDatadogTagsSize() { size = PTagsCodec.calcXDatadogTagsSize( size, ORG_PROPAGATION_MARKER_TAG, getOrgPropagationMarkerTagValue()); - size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_ML_APP_TAG, llmObsMlAppTagValue); + // One snapshot: sizing a mix of old and new values would gate the header on a tag set that + // never existed, and this total is what decides whether x-datadog-tags is emitted at all. + LLMObsTagValues currentLLMObsTags = llmObsTags; + size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_ML_APP_TAG, currentLLMObsTags.mlApp); size = - PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_SESSION_ID_TAG, llmObsSessionIdTagValue); + PTagsCodec.calcXDatadogTagsSize( + size, LLMOBS_SESSION_ID_TAG, currentLLMObsTags.sessionId); size = PTagsCodec.calcXDatadogTagsSize( - size, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsParentAgentSpanIdTagValue); + size, LLMOBS_PAGENT_SPAN_ID_TAG, currentLLMObsTags.parentAgentSpanId); size = PTagsCodec.calcXDatadogTagsSize( - size, LLMOBS_PAGENT_NAME_TAG, llmObsParentAgentNameTagValue); - size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_PARENT_ID_TAG, llmObsParentIdTagValue); + size, LLMOBS_PAGENT_NAME_TAG, currentLLMObsTags.parentAgentName); + size = + PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_PARENT_ID_TAG, currentLLMObsTags.parentId); int currentProductTraceSource = traceSource; if (currentProductTraceSource != ProductTraceSource.UNSET) { size = @@ -695,8 +657,7 @@ public void updateAndLockDecisionMaker(PropagationTags source) { canChangeDecisionMaker = false; decisionMakerTagValue = ((PTags) source).getDecisionMakerTagValue(); if (decisionMakerTagValue != null) { - clearCachedHeader(DATADOG); - clearCachedHeader(W3C); + clearCachedHeaders(); } } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index fc34fe043c0..8a56ab28a05 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -217,7 +217,7 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { maxUnknownSize, lastParentId, orgPropagationMarkerTagValue, - new LLMObsTagValues( + LLMObsTagValues.of( llmObsMlAppTagValue, llmObsSessionIdTagValue, llmObsParentAgentSpanIdTagValue, @@ -786,7 +786,7 @@ private static W3CPTags empty( 0, null, null, - null); + LLMObsTagValues.EMPTY); } private static class W3CPTags extends PTags { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 4273f75948b..b6141a6418a 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -63,8 +63,18 @@ default CharSequence getLLMObsMlApp() { return null; } - /** Sets the LLM Observability {@code ml_app} to propagate with this trace. No-op by default. */ - default void updateLLMObsMlApp(CharSequence mlApp) {} + /** + * Sets the whole LLM Observability tag set to propagate with this trace, replacing any set + * previously staged. Taken together rather than one tag at a time so the update is atomic: a + * concurrent reader never serializes a header mixing values from two different contexts. No-op by + * default. + */ + default void updateLLMObsContext( + CharSequence mlApp, + CharSequence sessionId, + CharSequence parentAgentSpanId, + CharSequence parentAgentName, + CharSequence parentId) {} /** * Gets the LLM Observability {@code session_id} propagated with this trace, or {@code null} if @@ -74,11 +84,6 @@ default CharSequence getLLMObsSessionId() { return null; } - /** - * Sets the LLM Observability {@code session_id} to propagate with this trace. No-op by default. - */ - default void updateLLMObsSessionId(CharSequence sessionId) {} - /** * Gets the span id of the parent LLM Observability agent span propagated with this trace, or * {@code null} if none is set or this context implementation doesn't have propagation-tags @@ -88,11 +93,6 @@ default CharSequence getLLMObsParentAgentSpanId() { return null; } - /** - * Sets the parent LLM Observability agent span id to propagate with this trace. No-op by default. - */ - default void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) {} - /** * Gets the name of the parent LLM Observability agent span propagated with this trace, or {@code * null} if none is set or this context implementation doesn't have propagation-tags access. @@ -101,12 +101,6 @@ default CharSequence getLLMObsParentAgentName() { return null; } - /** - * Sets the parent LLM Observability agent span name to propagate with this trace. No-op by - * default. - */ - default void updateLLMObsParentAgentName(CharSequence parentAgentName) {} - /** * Gets the span id of the parent LLM Observability span propagated with this trace, or {@code * null} if none is set or this context implementation doesn't have propagation-tags access. @@ -115,9 +109,6 @@ default CharSequence getLLMObsParentId() { return null; } - /** Sets the parent LLM Observability span id to propagate with this trace. No-op by default. */ - default void updateLLMObsParentId(CharSequence parentId) {} - /** * Gets whether the span context used is part of the local trace or from another service * From 3d9361fdd13ef528993e499c0e73165df2c833a6 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Tue, 8 Sep 2026 14:19:27 -0400 Subject: [PATCH 5/5] Inline the LLM Observability tag map writes to match the surrounding style Co-Authored-By: Claude Opus 5 --- .../core/propagation/ptags/PTagsCodec.java | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index 9869d42f635..ec865e79c6f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -160,22 +160,33 @@ static void fillTagMap(PTags propagationTags, Map tagMap) { .toString()); } LLMObsTagValues llmObsTags = propagationTags.getLLMObsTagValues(); - putLLMObsTag(tagMap, LLMOBS_ML_APP_TAG, llmObsTags.mlApp); - putLLMObsTag(tagMap, LLMOBS_SESSION_ID_TAG, llmObsTags.sessionId); - putLLMObsTag(tagMap, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsTags.parentAgentSpanId); - putLLMObsTag(tagMap, LLMOBS_PAGENT_NAME_TAG, llmObsTags.parentAgentName); - putLLMObsTag(tagMap, LLMOBS_PARENT_ID_TAG, llmObsTags.parentId); - if (propagationTags.getError() != null) { - tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); + if (llmObsTags.mlApp != null) { + tagMap.put( + LLMOBS_ML_APP_TAG.forType(Encoding.DATADOG).toString(), + llmObsTags.mlApp.forType(Encoding.DATADOG).toString()); } - } - - /** Adds one LLM Observability tag to the span's tag map, skipping it when unset. */ - private static void putLLMObsTag(Map tagMap, TagKey tagKey, TagValue tagValue) { - if (tagValue != null) { + if (llmObsTags.sessionId != null) { tagMap.put( - tagKey.forType(Encoding.DATADOG).toString(), - tagValue.forType(Encoding.DATADOG).toString()); + LLMOBS_SESSION_ID_TAG.forType(Encoding.DATADOG).toString(), + llmObsTags.sessionId.forType(Encoding.DATADOG).toString()); + } + if (llmObsTags.parentAgentSpanId != null) { + tagMap.put( + LLMOBS_PAGENT_SPAN_ID_TAG.forType(Encoding.DATADOG).toString(), + llmObsTags.parentAgentSpanId.forType(Encoding.DATADOG).toString()); + } + if (llmObsTags.parentAgentName != null) { + tagMap.put( + LLMOBS_PAGENT_NAME_TAG.forType(Encoding.DATADOG).toString(), + llmObsTags.parentAgentName.forType(Encoding.DATADOG).toString()); + } + if (llmObsTags.parentId != null) { + tagMap.put( + LLMOBS_PARENT_ID_TAG.forType(Encoding.DATADOG).toString(), + llmObsTags.parentId.forType(Encoding.DATADOG).toString()); + } + if (propagationTags.getError() != null) { + tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); } }