From 920e682347cbce6738bf60d781fd79db0d872085 Mon Sep 17 00:00:00 2001 From: Pujarini Mohapatra Date: Mon, 12 Oct 2026 14:55:55 +0100 Subject: [PATCH 1/4] Add model observation telemetry attributes Emit normalized inference operations, response model identity, cache token usage, agent type, and typed endpoint ports across core and supported framework mappings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../docs/design.md | 32 ++++++++-- .../observability/core/agent_details.py | 3 + .../observability/core/constants.py | 3 + .../core/inference_call_details.py | 12 +++- .../observability/core/inference_scope.py | 36 +++++++++--- .../core/middleware/baggage_builder.py | 6 ++ .../observability/core/opentelemetry_scope.py | 2 + .../core/trace_processor/util.py | 1 + .../extensions/langchain/utils.py | 22 ++++++- .../observability/extensions/openai/utils.py | 9 ++- .../core/test_baggage_builder.py | 3 + .../core/test_inference_scope.py | 58 +++++++++++++++++++ .../langchain/test_wrapper_langchain.py | 41 +++++++++++++ .../openai/test_wrapper_openaiagents.py | 42 ++++++++++++++ 14 files changed, 252 insertions(+), 18 deletions(-) diff --git a/libraries/microsoft-agents-a365-observability-core/docs/design.md b/libraries/microsoft-agents-a365-observability-core/docs/design.md index 570eeabd..4ee2fcf7 100644 --- a/libraries/microsoft-agents-a365-observability-core/docs/design.md +++ b/libraries/microsoft-agents-a365-observability-core/docs/design.md @@ -121,13 +121,23 @@ with InvokeAgentScope.start( Traces LLM/AI model inference calls: ```python -from microsoft_agents_a365.observability.core import InferenceScope, InferenceCallDetails, Request +from microsoft_agents_a365.observability.core import ( + InferenceCallDetails, + InferenceOperationType, + InferenceScope, + Request, + ServiceEndpoint, +) with InferenceScope.start( request=Request(content="Hello"), details=InferenceCallDetails( - model_name="gpt-4", - provider="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", + responseModel="gpt-4-2025-04-14", + cacheReadInputTokens=20, + endpoint=ServiceEndpoint(hostname="models.example.com", port=443), ), agent_details=agent_details, ) as scope: @@ -317,11 +327,21 @@ class SpanDetails: ```python @dataclass class InferenceCallDetails: - model_name: str | None - provider: str | None - # Additional inference metadata + operationName: InferenceOperationType + model: str | None + providerName: str + responseModel: str | None + inputTokens: int | None + outputTokens: int | None + cacheReadInputTokens: int | None + cacheWriteInputTokens: int | None + endpoint: ServiceEndpoint | None ``` +At least one of `model` or `responseModel` must be present. Inference spans use +the lowercase `chat` operation required by the OpenTelemetry schema and Agent365 +model observation pipeline. Endpoint ports and token counts are emitted as integers. + ### ToolCallDetails ```python diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py index 2061f415..58fef5ee 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py @@ -30,6 +30,9 @@ class AgentDetails: agent_platform_id: Optional[str] = None """Platform ID for the agent.""" + agent_type: Optional[str] = None + """Type classification for the agent.""" + tenant_id: Optional[str] = None """Tenant ID for the agent.""" diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/constants.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/constants.py index 85871ba8..3a174044 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/constants.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/constants.py @@ -37,6 +37,7 @@ GEN_AI_RESPONSE_MODEL_KEY = "gen_ai.response.model" GEN_AI_RESPONSE_ID_KEY = "gen_ai.response.id" GEN_AI_AGENT_ID_KEY = "gen_ai.agent.id" +GEN_AI_AGENT_TYPE_KEY = "gen_ai.agent.type" GEN_AI_AGENT_NAME_KEY = "gen_ai.agent.name" GEN_AI_AGENT_DESCRIPTION_KEY = "gen_ai.agent.description" GEN_AI_AGENT_VERSION_KEY = "gen_ai.agent.version" @@ -47,6 +48,8 @@ GEN_AI_TOKEN_TYPE_KEY = "gen_ai.token.type" GEN_AI_USAGE_INPUT_TOKENS_KEY = "gen_ai.usage.input_tokens" GEN_AI_USAGE_OUTPUT_TOKENS_KEY = "gen_ai.usage.output_tokens" +GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY = "gen_ai.usage.cache_read.input_tokens" +GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY = "gen_ai.usage.cache_write.input_tokens" GEN_AI_CHOICE = "gen_ai.choice" GEN_AI_PROVIDER_NAME_KEY = "gen_ai.provider.name" diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py index f2d8dff1..c315751a 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py @@ -12,10 +12,20 @@ class InferenceCallDetails: """Details of an inference call for generative AI operations.""" operationName: InferenceOperationType - model: str + model: str | None providerName: str + responseModel: str | None = None inputTokens: int | None = None outputTokens: int | None = None + cacheReadInputTokens: int | None = None + cacheWriteInputTokens: int | None = None finishReasons: list[str] | None = None thoughtProcess: str | None = None endpoint: ServiceEndpoint | None = None + + def __post_init__(self) -> None: + """Validate required model identity.""" + if not (self.model and self.model.strip()) and not ( + self.responseModel and self.responseModel.strip() + ): + raise ValueError("At least one of model or responseModel must be provided") diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py index dd21dde0..7884dd6f 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py @@ -1,15 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from typing import List - from opentelemetry.trace import SpanKind from .agent_details import AgentDetails from .constants import ( CHANNEL_LINK_KEY, CHANNEL_NAME_KEY, + CHAT_OPERATION_NAME, GEN_AI_AGENT_THOUGHT_PROCESS_KEY, + GEN_AI_CALLER_CLIENT_IP_KEY, GEN_AI_CONVERSATION_ID_KEY, GEN_AI_INPUT_MESSAGES_KEY, GEN_AI_OPERATION_NAME_KEY, @@ -17,6 +17,9 @@ GEN_AI_PROVIDER_NAME_KEY, GEN_AI_REQUEST_MODEL_KEY, GEN_AI_RESPONSE_FINISH_REASONS_KEY, + GEN_AI_RESPONSE_MODEL_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, GEN_AI_USAGE_INPUT_TOKENS_KEY, GEN_AI_USAGE_OUTPUT_TOKENS_KEY, SERVER_ADDRESS_KEY, @@ -24,7 +27,6 @@ USER_EMAIL_KEY, USER_ID_KEY, USER_NAME_KEY, - GEN_AI_CALLER_CLIENT_IP_KEY, ) from .inference_call_details import InferenceCallDetails from .message_utils import ( @@ -95,9 +97,10 @@ def __init__( else SpanDetails(span_kind=SpanKind.CLIENT) ) + model_name = details.model or details.responseModel super().__init__( - operation_name=details.operationName.value, - activity_name=f"{details.operationName.value} {details.model}", + operation_name=CHAT_OPERATION_NAME, + activity_name=f"{CHAT_OPERATION_NAME} {model_name}", agent_details=agent_details, span_details=resolved_span_details, ) @@ -106,8 +109,9 @@ def __init__( self.record_input_messages(request.content) self.set_tag_maybe(GEN_AI_CONVERSATION_ID_KEY, request.conversation_id) - self.set_tag_maybe(GEN_AI_OPERATION_NAME_KEY, details.operationName.value) + self.set_tag_maybe(GEN_AI_OPERATION_NAME_KEY, CHAT_OPERATION_NAME) self.set_tag_maybe(GEN_AI_REQUEST_MODEL_KEY, details.model) + self.set_tag_maybe(GEN_AI_RESPONSE_MODEL_KEY, details.responseModel) self.set_tag_maybe(GEN_AI_PROVIDER_NAME_KEY, details.providerName) self.set_tag_maybe( GEN_AI_USAGE_INPUT_TOKENS_KEY, @@ -117,6 +121,14 @@ def __init__( GEN_AI_USAGE_OUTPUT_TOKENS_KEY, details.outputTokens if details.outputTokens is not None else None, ) + self.set_tag_maybe( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + details.cacheReadInputTokens, + ) + self.set_tag_maybe( + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + details.cacheWriteInputTokens, + ) self.set_tag_maybe( GEN_AI_RESPONSE_FINISH_REASONS_KEY, safe_json_dumps(details.finishReasons) if details.finishReasons else None, @@ -127,7 +139,7 @@ def __init__( if details.endpoint: self.set_tag_maybe(SERVER_ADDRESS_KEY, details.endpoint.hostname) if details.endpoint.port: - self.set_tag_maybe(SERVER_PORT_KEY, str(details.endpoint.port)) + self.set_tag_maybe(SERVER_PORT_KEY, details.endpoint.port) # Set request metadata if provided if request.channel: @@ -184,7 +196,15 @@ def record_output_tokens(self, output_tokens: int) -> None: """ self.set_tag_maybe(GEN_AI_USAGE_OUTPUT_TOKENS_KEY, output_tokens) - def record_finish_reasons(self, finish_reasons: List[str]) -> None: + def record_cache_read_input_tokens(self, input_tokens: int) -> None: + """Record the number of input tokens read from a cache.""" + self.set_tag_maybe(GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, input_tokens) + + def record_cache_write_input_tokens(self, input_tokens: int) -> None: + """Record the number of input tokens written to a cache.""" + self.set_tag_maybe(GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, input_tokens) + + def record_finish_reasons(self, finish_reasons: list[str]) -> None: """Records the finish reasons for telemetry tracking. Args: diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py index 5e0a638d..1bfb9d6e 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py @@ -17,6 +17,7 @@ GEN_AI_AGENT_EMAIL_KEY, GEN_AI_AGENT_ID_KEY, GEN_AI_AGENT_NAME_KEY, + GEN_AI_AGENT_TYPE_KEY, GEN_AI_AGENT_VERSION_KEY, GEN_AI_CALLER_CLIENT_IP_KEY, GEN_AI_CONVERSATION_ID_KEY, @@ -97,6 +98,11 @@ def agent_id(self, value: str | None) -> "BaggageBuilder": self._set(GEN_AI_AGENT_ID_KEY, value) return self + def agent_type(self, value: str | None) -> "BaggageBuilder": + """Set the agent type baggage value.""" + self._set(GEN_AI_AGENT_TYPE_KEY, value) + return self + def agentic_user_id(self, value: str | None) -> "BaggageBuilder": """Set the agentic user ID baggage value. diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py index ff40360f..71c07594 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py @@ -33,6 +33,7 @@ GEN_AI_AGENT_ID_KEY, GEN_AI_AGENT_NAME_KEY, GEN_AI_AGENT_PLATFORM_ID_KEY, + GEN_AI_AGENT_TYPE_KEY, GEN_AI_AGENT_VERSION_KEY, GEN_AI_ICON_URI_KEY, GEN_AI_OPERATION_NAME_KEY, @@ -189,6 +190,7 @@ def __init__( self.set_tag_maybe( GEN_AI_AGENT_PLATFORM_ID_KEY, agent_details.agent_platform_id ) + self.set_tag_maybe(GEN_AI_AGENT_TYPE_KEY, agent_details.agent_type) self.set_tag_maybe(TENANT_ID_KEY, agent_details.tenant_id) self.set_tag_maybe(GEN_AI_ICON_URI_KEY, agent_details.icon_uri) # Set provider name dynamically from agent details diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/trace_processor/util.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/trace_processor/util.py index bef32cd9..7faeca2a 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/trace_processor/util.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/trace_processor/util.py @@ -12,6 +12,7 @@ consts.GEN_AI_CONVERSATION_ITEM_LINK_KEY, # microsoft.conversation.item.link consts.GEN_AI_OPERATION_NAME_KEY, # gen_ai.operation.name consts.GEN_AI_AGENT_ID_KEY, # gen_ai.agent.id + consts.GEN_AI_AGENT_TYPE_KEY, # gen_ai.agent.type consts.GEN_AI_AGENT_NAME_KEY, # gen_ai.agent.name consts.GEN_AI_AGENT_DESCRIPTION_KEY, # gen_ai.agent.description consts.GEN_AI_AGENT_VERSION_KEY, # gen_ai.agent.version diff --git a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py index 6114dba7..2e4b2d69 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py +++ b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py @@ -17,6 +17,7 @@ GEN_AI_REQUEST_MODEL_KEY, GEN_AI_RESPONSE_FINISH_REASONS_KEY, GEN_AI_RESPONSE_ID_KEY, + GEN_AI_RESPONSE_MODEL_KEY, GEN_AI_SYSTEM_INSTRUCTIONS_KEY, GEN_AI_TOOL_ARGS_KEY, GEN_AI_TOOL_CALL_ID_KEY, @@ -24,6 +25,8 @@ GEN_AI_TOOL_DESCRIPTION_KEY, GEN_AI_TOOL_NAME_KEY, GEN_AI_TOOL_TYPE_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, GEN_AI_USAGE_INPUT_TOKENS_KEY, GEN_AI_USAGE_OUTPUT_TOKENS_KEY, INVOKE_AGENT_OPERATION_NAME, @@ -299,8 +302,8 @@ def model_name( ): for key in "model_name", "model": if name := str(llm_output.get(key) or "").strip(): - yield GEN_AI_REQUEST_MODEL_KEY, name - return + yield GEN_AI_RESPONSE_MODEL_KEY, name + break if not extra: return if not isinstance(extra, Mapping): @@ -348,6 +351,11 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] # OpenAI for attribute_name, details_key, keys in [ + ( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + "prompt_tokens_details", + ("cached_tokens",), + ), ( GEN_AI_RESPONSE_FINISH_REASONS_KEY, "completion_tokens_details", @@ -362,6 +370,16 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] for attribute_name, details_key_or_none, keys in [ (GEN_AI_USAGE_INPUT_TOKENS_KEY, None, ("input_tokens",)), (GEN_AI_USAGE_OUTPUT_TOKENS_KEY, None, ("output_tokens",)), + ( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + "input_token_details", + ("cache_read", "cached_tokens"), + ), + ( + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + "input_token_details", + ("cache_write", "cache_creation"), + ), ( GEN_AI_RESPONSE_FINISH_REASONS_KEY, "output_token_details", diff --git a/libraries/microsoft-agents-a365-observability-extensions-openai/microsoft_agents_a365/observability/extensions/openai/utils.py b/libraries/microsoft-agents-a365-observability-extensions-openai/microsoft_agents_a365/observability/extensions/openai/utils.py index f32af728..170a1d99 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-openai/microsoft_agents_a365/observability/extensions/openai/utils.py +++ b/libraries/microsoft-agents-a365-observability-extensions-openai/microsoft_agents_a365/observability/extensions/openai/utils.py @@ -28,10 +28,12 @@ GEN_AI_PROVIDER_NAME_KEY, GEN_AI_REQUEST_MODEL_KEY, GEN_AI_RESPONSE_FINISH_REASONS_KEY, + GEN_AI_RESPONSE_MODEL_KEY, GEN_AI_TOOL_ARGS_KEY, GEN_AI_TOOL_CALL_ID_KEY, GEN_AI_TOOL_CALL_RESULT_KEY, GEN_AI_TOOL_NAME_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, GEN_AI_USAGE_INPUT_TOKENS_KEY, GEN_AI_USAGE_OUTPUT_TOKENS_KEY, ) @@ -389,7 +391,7 @@ def get_attributes_from_response(obj: Response) -> Iterator[tuple[str, Attribute yield from _get_attributes_from_response_instruction(obj.instructions) else: pass # TODO: handle list instructions - yield GEN_AI_REQUEST_MODEL_KEY, obj.model + yield GEN_AI_RESPONSE_MODEL_KEY, obj.model param = obj.model_dump( exclude_none=True, exclude={"object", "tools", "usage", "output", "error", "status"}, @@ -512,6 +514,11 @@ def get_attributes_from_usage( yield GEN_AI_USAGE_INPUT_TOKENS_KEY, obj.input_tokens yield GEN_AI_LLM_TOKEN_COUNT_TOTAL, obj.total_tokens yield GEN_AI_LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHED_READ, obj.input_tokens_details.cached_tokens + if obj.input_tokens_details.cached_tokens is not None: + yield ( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + obj.input_tokens_details.cached_tokens, + ) yield ( GEN_AI_LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, obj.output_tokens_details.reasoning_tokens, diff --git a/tests/observability/core/test_baggage_builder.py b/tests/observability/core/test_baggage_builder.py index e4a77e7c..ec3862d4 100644 --- a/tests/observability/core/test_baggage_builder.py +++ b/tests/observability/core/test_baggage_builder.py @@ -11,6 +11,7 @@ GEN_AI_AGENT_BLUEPRINT_ID_KEY, GEN_AI_AGENT_EMAIL_KEY, GEN_AI_AGENT_ID_KEY, + GEN_AI_AGENT_TYPE_KEY, GEN_AI_AGENT_VERSION_KEY, GEN_AI_CALLER_CLIENT_IP_KEY, SERVER_ADDRESS_KEY, @@ -86,6 +87,7 @@ def test_all_baggage_keys(self): BaggageBuilder() .tenant_id("tenant-1") .agent_id("agent-1") + .agent_type("Foundry") .agentic_user_id("auid-1") .agentic_user_email("upn-1") .agent_blueprint_id("blueprint-1") @@ -96,6 +98,7 @@ def test_all_baggage_keys(self): current_baggage = baggage.get_all() self.assertEqual(current_baggage.get(TENANT_ID_KEY), "tenant-1") self.assertEqual(current_baggage.get(GEN_AI_AGENT_ID_KEY), "agent-1") + self.assertEqual(current_baggage.get(GEN_AI_AGENT_TYPE_KEY), "Foundry") self.assertEqual(current_baggage.get(GEN_AI_AGENT_AUID_KEY), "auid-1") self.assertEqual(current_baggage.get(GEN_AI_AGENT_EMAIL_KEY), "upn-1") self.assertEqual(current_baggage.get(GEN_AI_AGENT_BLUEPRINT_ID_KEY), "blueprint-1") diff --git a/tests/observability/core/test_inference_scope.py b/tests/observability/core/test_inference_scope.py index 5722c845..6057cc59 100644 --- a/tests/observability/core/test_inference_scope.py +++ b/tests/observability/core/test_inference_scope.py @@ -13,6 +13,7 @@ InferenceOperationType, InferenceScope, Request, + ServiceEndpoint, SpanDetails, configure, extract_context_from_headers, @@ -23,6 +24,18 @@ from microsoft_agents_a365.observability.core.constants import ( CHANNEL_LINK_KEY, CHANNEL_NAME_KEY, + CHAT_OPERATION_NAME, + GEN_AI_AGENT_ID_KEY, + GEN_AI_AGENT_TYPE_KEY, + GEN_AI_OPERATION_NAME_KEY, + GEN_AI_PROVIDER_NAME_KEY, + GEN_AI_RESPONSE_MODEL_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_OUTPUT_TOKENS_KEY, + SERVER_ADDRESS_KEY, + SERVER_PORT_KEY, ) from microsoft_agents_a365.observability.core.opentelemetry_scope import OpenTelemetryScope from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -109,6 +122,51 @@ def test_inference_call_details_with_all_fields(self): self.assertEqual(details.outputTokens, 75) self.assertEqual(details.finishReasons, ["stop"]) + def test_inference_call_details_requires_request_or_response_model(self): + """Test that an inference call always has model identity.""" + with self.assertRaisesRegex(ValueError, "model or responseModel"): + InferenceCallDetails( + operationName=InferenceOperationType.CHAT, + model=None, + providerName="openai", + ) + + def test_inference_scope_emits_model_observation_attributes(self): + """Test the attributes required by the model observation pipeline.""" + details = InferenceCallDetails( + operationName=InferenceOperationType.TEXT_COMPLETION, + model=None, + responseModel="gpt-4.1-2025-04-14", + providerName="openai", + inputTokens=150, + outputTokens=75, + cacheReadInputTokens=30, + cacheWriteInputTokens=10, + endpoint=ServiceEndpoint(hostname="models.example.com", port=443), + ) + agent_details = AgentDetails( + agent_id="test-inference-agent", + agent_type="Foundry", + ) + + scope = InferenceScope.start(Request(), details, agent_details) + scope.dispose() + + span = self.span_exporter.get_finished_spans()[-1] + attributes = span.attributes or {} + + self.assertEqual(attributes[GEN_AI_OPERATION_NAME_KEY], CHAT_OPERATION_NAME) + self.assertEqual(attributes[GEN_AI_PROVIDER_NAME_KEY], "openai") + self.assertEqual(attributes[GEN_AI_RESPONSE_MODEL_KEY], "gpt-4.1-2025-04-14") + self.assertEqual(attributes[GEN_AI_AGENT_ID_KEY], "test-inference-agent") + self.assertEqual(attributes[GEN_AI_AGENT_TYPE_KEY], "Foundry") + self.assertEqual(attributes[SERVER_ADDRESS_KEY], "models.example.com") + self.assertEqual(attributes[SERVER_PORT_KEY], 443) + self.assertEqual(attributes[GEN_AI_USAGE_INPUT_TOKENS_KEY], 150) + self.assertEqual(attributes[GEN_AI_USAGE_OUTPUT_TOKENS_KEY], 75) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 30) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + def test_inference_scope_start_method(self): """Test InferenceScope.start() static method.""" details = InferenceCallDetails( diff --git a/tests/observability/extensions/langchain/test_wrapper_langchain.py b/tests/observability/extensions/langchain/test_wrapper_langchain.py index 0a9c1116..7ab01572 100644 --- a/tests/observability/extensions/langchain/test_wrapper_langchain.py +++ b/tests/observability/extensions/langchain/test_wrapper_langchain.py @@ -6,9 +6,50 @@ from uuid import uuid4 from microsoft_agents_a365.observability.core import configure +from microsoft_agents_a365.observability.core.constants import ( + GEN_AI_REQUEST_MODEL_KEY, + GEN_AI_RESPONSE_MODEL_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, +) from microsoft_agents_a365.observability.extensions.langchain.tracer_instrumentor import ( CustomLangChainInstrumentor, ) +from microsoft_agents_a365.observability.extensions.langchain.utils import model_name, token_counts + + +class TestLangChainModelObservationAttributes(unittest.TestCase): + """Unit tests for model observation attribute mapping.""" + + def test_request_and_response_models_are_distinct(self): + """Test that requested and served model names use distinct attributes.""" + outputs = {"llm_output": {"model_name": "gpt-4.1-2025-04-14"}} + extra = {"metadata": {"ls_model_name": "gpt-4.1"}} + + attributes = dict(model_name(outputs, extra)) + + self.assertEqual(attributes[GEN_AI_REQUEST_MODEL_KEY], "gpt-4.1") + self.assertEqual(attributes[GEN_AI_RESPONSE_MODEL_KEY], "gpt-4.1-2025-04-14") + + def test_cache_token_counts_use_standard_attributes(self): + """Test that cache token counts use model observation attributes.""" + outputs = { + "llm_output": { + "token_usage": { + "input_tokens": 100, + "output_tokens": 20, + "input_token_details": { + "cache_read": 40, + "cache_creation": 10, + }, + } + } + } + + attributes = dict(token_counts(outputs)) + + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) class TestInstrumentorLangChain(unittest.TestCase): diff --git a/tests/observability/extensions/openai/test_wrapper_openaiagents.py b/tests/observability/extensions/openai/test_wrapper_openaiagents.py index b95c99ce..ffc14a00 100644 --- a/tests/observability/extensions/openai/test_wrapper_openaiagents.py +++ b/tests/observability/extensions/openai/test_wrapper_openaiagents.py @@ -2,9 +2,51 @@ # Licensed under the MIT License. import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock from microsoft_agents_a365.observability.core import configure +from microsoft_agents_a365.observability.core.constants import ( + GEN_AI_RESPONSE_MODEL_KEY, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, +) from microsoft_agents_a365.observability.extensions.openai import OpenAIAgentsTraceInstrumentor +from microsoft_agents_a365.observability.extensions.openai.utils import ( + get_attributes_from_response, + get_attributes_from_usage, +) + + +class TestOpenAIModelObservationAttributes(unittest.TestCase): + """Unit tests for model observation attribute mapping.""" + + def test_response_model_uses_response_attribute(self): + """Test that the served model is emitted as the response model.""" + response = MagicMock() + response.tools = [] + response.usage = None + response.output = [] + response.instructions = None + response.model = "gpt-4.1-2025-04-14" + response.model_dump.return_value = {} + + attributes = dict(get_attributes_from_response(response)) + + self.assertEqual(attributes[GEN_AI_RESPONSE_MODEL_KEY], "gpt-4.1-2025-04-14") + + def test_cached_tokens_use_standard_attribute(self): + """Test that cached input tokens use the model observation attribute.""" + usage = SimpleNamespace( + input_tokens=100, + output_tokens=20, + total_tokens=120, + input_tokens_details=SimpleNamespace(cached_tokens=40), + output_tokens_details=SimpleNamespace(reasoning_tokens=5), + ) + + attributes = dict(get_attributes_from_usage(usage)) + + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) class TestOpenAIAgentsTraceInstrumentor(unittest.TestCase): From 4eaf0a06e54848742c82cfbb92bcead987d57580 Mon Sep 17 00:00:00 2001 From: Pujarini Mohapatra Date: Thu, 15 Oct 2026 12:06:07 +0100 Subject: [PATCH 2/4] Fix PR #282 review feedback: dataclass compatibility, operation naming, cache tokens Addresses 6 unresolved review threads on PR #282: 1. AgentDetails.agent_type: moved to the end of the dataclass so existing positional callers aren't shifted; type widened to Optional[Union[AgentType, str]]. 2. InferenceCallDetails: moved responseModel/cacheReadInputTokens/ cacheWriteInputTokens to the end of the dataclass to preserve positional constructor compatibility. 3. LangChain utils.token_counts(): now reads normalized cache token details from the generated AIMessage's usage_metadata (both direct attribute and dumpd kwargs-serialized forms) before falling back to llm_output-based token_usage, and also handles Anthropic's flat raw cache_read_input_tokens/cache_creation_input_tokens fields. Added tests using the real message shape instead of a synthetic llm_output shape. 4. InferenceCallDetails.__post_init__: normalizes model/responseModel by stripping whitespace and treating blank values as None, so a whitespace-only model no longer produces a blank gen_ai.request.model / malformed span name when responseModel is set. 5. AgentType enum handling: OpenTelemetryScope and BaggageBuilder.agent_type now accept AgentType | str | None and emit .value for enum members so OTel doesn't silently drop AgentType.FOUNDRY-style values. 6. InferenceScope operation naming: added InferenceOperationType.canonical_name (maps CHAT/TEXT_COMPLETION/GENERATE_CONTENT to their canonical lowercase semconv spelling) and use it for both the span name prefix and gen_ai.operation.name, replacing the hardcoded "chat" value. Updated the Agent365 exporter's GEN_AI_OPERATION_NAMES allowlist to include the canonical lowercase names (kept legacy PascalCase entries), fixed the PR's own test that wrongly asserted "chat" for a TEXT_COMPLETION input, and updated docs/design.md accordingly. Also added CHANGELOG.md entries for both affected packages. All changes verified with the full core + langchain test suites (278 passed, plus the 5 newly added tests) via `pytest tests/observability/core/ tests/observability/extensions/langchain/ -m "not integration"`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CHANGELOG.md | 24 ++++ .../docs/design.md | 20 ++-- .../observability/core/agent_details.py | 14 ++- .../observability/core/exporters/utils.py | 9 ++ .../core/inference_call_details.py | 24 ++-- .../core/inference_operation_type.py | 9 ++ .../observability/core/inference_scope.py | 8 +- .../core/middleware/baggage_builder.py | 12 +- .../observability/core/opentelemetry_scope.py | 8 +- .../CHANGELOG.md | 10 ++ .../extensions/langchain/utils.py | 107 ++++++++++++++---- .../core/test_inference_scope.py | 75 +++++++++++- .../langchain/test_wrapper_langchain.py | 74 ++++++++++++ 13 files changed, 345 insertions(+), 49 deletions(-) diff --git a/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md b/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md index 1159082d..092a6274 100644 --- a/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md +++ b/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to this package will be documented in this file. ## [Unreleased] +### Fixed + +- **`InferenceScope` span name and `gen_ai.operation.name` now reflect the actual operation** — + Previously every inference span was hardcoded to `chat`, regardless of + `InferenceCallDetails.operationName`. The span name and the + `gen_ai.operation.name` attribute are now derived from the canonical + lowercase semconv value of the operation (e.g. `chat`, `text_completion`, + `generate_content`), matching the pre-existing Node.js/.NET behavior. +- **Agent 365 exporter's `GEN_AI_OPERATION_NAMES` allowlist now recognizes the + canonical lowercase operation names** (`chat`, `text_completion`, + `generate_content`) in addition to the existing aliases, since spans can now + legitimately report these values. +- **`AgentDetails.agent_type` and `InferenceCallDetails.responseModel`/cache + fields are appended at the end of their dataclasses** instead of being + inserted in the middle, preserving positional-argument compatibility for + existing callers. +- **Whitespace-only `model`/`responseModel` values are now normalized to + `None`** in `InferenceCallDetails.__post_init__`, preventing blank span + names/`gen_ai.request.model` attributes when a non-blank `responseModel` is + also supplied. +- **`AgentDetails.agent_type` now accepts `AgentType` enum members** — the + enum's `.value` is emitted on the span/baggage instead of the enum object, + which OpenTelemetry would otherwise silently drop. + ### Breaking Changes - **OBS exports always use `/observabilityService`** — The `use_s2s_endpoint` diff --git a/libraries/microsoft-agents-a365-observability-core/docs/design.md b/libraries/microsoft-agents-a365-observability-core/docs/design.md index 4ee2fcf7..e4710162 100644 --- a/libraries/microsoft-agents-a365-observability-core/docs/design.md +++ b/libraries/microsoft-agents-a365-observability-core/docs/design.md @@ -286,9 +286,9 @@ class AgentDetails: agent_auid: str | None # Agent unique identifier agent_email: str | None # Agent email address agent_blueprint_id: str | None - agent_type: AgentType | None tenant_id: str | None icon_uri: str | None + agent_type: AgentType | str | None # appended last for positional compat ``` ### UserDetails @@ -330,17 +330,23 @@ class InferenceCallDetails: operationName: InferenceOperationType model: str | None providerName: str - responseModel: str | None inputTokens: int | None outputTokens: int | None - cacheReadInputTokens: int | None - cacheWriteInputTokens: int | None + finishReasons: list[str] | None + thoughtProcess: str | None endpoint: ServiceEndpoint | None + responseModel: str | None # appended last for positional compat + cacheReadInputTokens: int | None # appended last for positional compat + cacheWriteInputTokens: int | None # appended last for positional compat ``` -At least one of `model` or `responseModel` must be present. Inference spans use -the lowercase `chat` operation required by the OpenTelemetry schema and Agent365 -model observation pipeline. Endpoint ports and token counts are emitted as integers. +At least one of `model` or `responseModel` must be present (whitespace-only +values are normalized to `None`). The `gen_ai.operation.name` attribute and the +span name prefix are derived from `operationName` via its canonical lowercase +form (`InferenceOperationType.CHAT` -> `chat`, `TEXT_COMPLETION` -> +`text_completion`, `GENERATE_CONTENT` -> `generate_content`), matching the +OpenTelemetry GenAI semantic conventions and the Agent365 model observation +pipeline. Endpoint ports and token counts are emitted as integers. ### ToolCallDetails diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py index 58fef5ee..0a4e8752 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/agent_details.py @@ -2,7 +2,9 @@ # Licensed under the MIT License. from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union + +from .models.agent_type import AgentType @dataclass @@ -30,9 +32,6 @@ class AgentDetails: agent_platform_id: Optional[str] = None """Platform ID for the agent.""" - agent_type: Optional[str] = None - """Type classification for the agent.""" - tenant_id: Optional[str] = None """Tenant ID for the agent.""" @@ -44,3 +43,10 @@ class AgentDetails: agent_version: Optional[str] = None """Optional version of the agent (e.g., "1.0.0", "2025-05-01").""" + + # NOTE: Appended after all pre-existing fields (rather than inserted in the + # middle) to preserve positional-argument compatibility for existing callers. + agent_type: Optional[Union[AgentType, str]] = None + """Type classification for the agent. Accepts an `AgentType` enum member or + a raw string; enum members are normalized to their string `.value` when + emitted as a span attribute or baggage entry.""" diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/exporters/utils.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/exporters/utils.py index d3f140e2..50c02a4e 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/exporters/utils.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/exporters/utils.py @@ -34,6 +34,12 @@ # Operation names that identify a span as eligible for export to the Agent 365 # observability ingest service. Only spans whose gen_ai.operation.name matches # one of these values are included; all other spans are filtered out. +# +# Inference spans now emit the canonical lowercase operation name for their +# InferenceOperationType (e.g. "chat", "text_completion", "generate_content" - +# see InferenceOperationType.canonical_name), so all three are recognized here. +# The legacy PascalCase "Chat" value is kept for back-compat with any +# previously-emitted/non-Python spans that still use the raw enum value. GEN_AI_OPERATION_NAMES: frozenset[str] = frozenset( { INVOKE_AGENT_OPERATION_NAME, @@ -42,6 +48,9 @@ APPLY_GUARDRAIL_OPERATION_NAME, CHAT_OPERATION_NAME, InferenceOperationType.CHAT.value, + InferenceOperationType.CHAT.canonical_name, + InferenceOperationType.TEXT_COMPLETION.canonical_name, + InferenceOperationType.GENERATE_CONTENT.canonical_name, } ) diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py index c315751a..3d41cfc5 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_call_details.py @@ -14,18 +14,28 @@ class InferenceCallDetails: operationName: InferenceOperationType model: str | None providerName: str - responseModel: str | None = None inputTokens: int | None = None outputTokens: int | None = None - cacheReadInputTokens: int | None = None - cacheWriteInputTokens: int | None = None finishReasons: list[str] | None = None thoughtProcess: str | None = None endpoint: ServiceEndpoint | None = None + # NOTE: Appended after all pre-existing fields (rather than inserted in the + # middle) to preserve positional-argument compatibility for existing callers. + responseModel: str | None = None + cacheReadInputTokens: int | None = None + cacheWriteInputTokens: int | None = None def __post_init__(self) -> None: - """Validate required model identity.""" - if not (self.model and self.model.strip()) and not ( - self.responseModel and self.responseModel.strip() - ): + """Normalize and validate required model identity. + + Whitespace-only `model`/`responseModel` values are normalized to + `None` so that validation and downstream consumers (e.g. + `InferenceScope`) never treat a blank string as a present model. + """ + if self.model is not None: + self.model = self.model.strip() or None + if self.responseModel is not None: + self.responseModel = self.responseModel.strip() or None + + if not self.model and not self.responseModel: raise ValueError("At least one of model or responseModel must be provided") diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_operation_type.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_operation_type.py index 1b2809c2..a50ea35a 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_operation_type.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_operation_type.py @@ -10,3 +10,12 @@ class InferenceOperationType(Enum): CHAT = "Chat" TEXT_COMPLETION = "TextCompletion" GENERATE_CONTENT = "GenerateContent" + + @property + def canonical_name(self) -> str: + """The canonical lowercase operation name per the OpenTelemetry GenAI + semantic conventions (e.g. ``chat``, ``text_completion``, + ``generate_content``). Used for both the span name prefix and the + ``gen_ai.operation.name`` attribute. + """ + return self.name.lower() diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py index 7884dd6f..800637e0 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py @@ -7,7 +7,6 @@ from .constants import ( CHANNEL_LINK_KEY, CHANNEL_NAME_KEY, - CHAT_OPERATION_NAME, GEN_AI_AGENT_THOUGHT_PROCESS_KEY, GEN_AI_CALLER_CLIENT_IP_KEY, GEN_AI_CONVERSATION_ID_KEY, @@ -98,9 +97,10 @@ def __init__( ) model_name = details.model or details.responseModel + operation_name = details.operationName.canonical_name super().__init__( - operation_name=CHAT_OPERATION_NAME, - activity_name=f"{CHAT_OPERATION_NAME} {model_name}", + operation_name=operation_name, + activity_name=f"{operation_name} {model_name}", agent_details=agent_details, span_details=resolved_span_details, ) @@ -109,7 +109,7 @@ def __init__( self.record_input_messages(request.content) self.set_tag_maybe(GEN_AI_CONVERSATION_ID_KEY, request.conversation_id) - self.set_tag_maybe(GEN_AI_OPERATION_NAME_KEY, CHAT_OPERATION_NAME) + self.set_tag_maybe(GEN_AI_OPERATION_NAME_KEY, operation_name) self.set_tag_maybe(GEN_AI_REQUEST_MODEL_KEY, details.model) self.set_tag_maybe(GEN_AI_RESPONSE_MODEL_KEY, details.responseModel) self.set_tag_maybe(GEN_AI_PROVIDER_NAME_KEY, details.providerName) diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py index 1bfb9d6e..6845f6e2 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/middleware/baggage_builder.py @@ -32,6 +32,7 @@ USER_ID_KEY, USER_NAME_KEY, ) +from ..models.agent_type import AgentType from ..utils import validate_and_normalize_ip logger = logging.getLogger(__name__) @@ -98,9 +99,14 @@ def agent_id(self, value: str | None) -> "BaggageBuilder": self._set(GEN_AI_AGENT_ID_KEY, value) return self - def agent_type(self, value: str | None) -> "BaggageBuilder": - """Set the agent type baggage value.""" - self._set(GEN_AI_AGENT_TYPE_KEY, value) + def agent_type(self, value: AgentType | str | None) -> "BaggageBuilder": + """Set the agent type baggage value. + + Accepts either an `AgentType` enum member or a plain string; enum + members are normalized to their string `.value` since OTel baggage + values must be primitive strings. + """ + self._set(GEN_AI_AGENT_TYPE_KEY, getattr(value, "value", value)) return self def agentic_user_id(self, value: str | None) -> "BaggageBuilder": diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py index 71c07594..a7d40cc7 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/opentelemetry_scope.py @@ -190,7 +190,13 @@ def __init__( self.set_tag_maybe( GEN_AI_AGENT_PLATFORM_ID_KEY, agent_details.agent_platform_id ) - self.set_tag_maybe(GEN_AI_AGENT_TYPE_KEY, agent_details.agent_type) + # Accept either an `AgentType` enum member or a plain string; + # emit the string `.value` so OTel doesn't silently drop + # non-primitive attribute values. + self.set_tag_maybe( + GEN_AI_AGENT_TYPE_KEY, + getattr(agent_details.agent_type, "value", agent_details.agent_type), + ) self.set_tag_maybe(TENANT_ID_KEY, agent_details.tenant_id) self.set_tag_maybe(GEN_AI_ICON_URI_KEY, agent_details.icon_uri) # Set provider name dynamically from agent details diff --git a/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md b/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md index 1452eaa5..f58dc5aa 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md +++ b/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this package will be documented in this file. ## [Unreleased] +### Fixed + +- **Cache token counts now read the generated message's normalized + `usage_metadata`** (`gen_ai.usage.cache_read_input_tokens` / + `gen_ai.usage.cache_write_input_tokens`), falling back to the + provider-specific `llm_output` usage payload. Previously only + `llm_output.token_usage`/`llm_output.usage` were checked, which missed + providers (e.g. ChatAnthropic) that normalize cache details onto the + generated `AIMessage` instead. + ### Changed - Removed `set_execution_type()` function and all `ExecutionType` / `GEN_AI_EXECUTION_TYPE_KEY` usage. diff --git a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py index 2e4b2d69..30a2941c 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py +++ b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py @@ -326,8 +326,11 @@ def model_name( @stop_on_exception def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]]: """Yields token count information if present.""" - if not (token_usage := (parse_token_usage_for_non_streaming_outputs(outputs))): + token_usage = parse_token_usage_for_non_streaming_outputs(outputs) + message_usage = _message_usage_metadata(outputs) + if not token_usage and not message_usage: return + for attribute_name, keys in [ ( GEN_AI_USAGE_INPUT_TOKENS_KEY, @@ -346,27 +349,44 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] ), ), ]: - if (token_count := get_first_value(token_usage, keys)) is not None: + if token_usage is not None and (token_count := get_first_value(token_usage, keys)) is not None: yield attribute_name, token_count - # OpenAI - for attribute_name, details_key, keys in [ - ( - GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, - "prompt_tokens_details", - ("cached_tokens",), - ), - ( - GEN_AI_RESPONSE_FINISH_REASONS_KEY, - "completion_tokens_details", - ("reasoning_tokens",), - ), - ]: - if (details := token_usage.get(details_key)) is not None: - if (token_count := get_first_value(details, keys)) is not None: + if token_usage is not None: + # OpenAI + for attribute_name, details_key, keys in [ + ( + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + "prompt_tokens_details", + ("cached_tokens",), + ), + ( + GEN_AI_RESPONSE_FINISH_REASONS_KEY, + "completion_tokens_details", + ("reasoning_tokens",), + ), + ]: + if (details := token_usage.get(details_key)) is not None: + if (token_count := get_first_value(details, keys)) is not None: + yield attribute_name, token_count + + # Anthropic's raw (non-normalized) usage payload places cache counts + # as flat fields instead of nesting them under `input_token_details`, + # e.g. llm_output["usage"]["cache_read_input_tokens"]. + for attribute_name, keys in [ + (GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, ("cache_read_input_tokens",)), + (GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, ("cache_creation_input_tokens",)), + ]: + if (token_count := get_first_value(token_usage, keys)) is not None: yield attribute_name, token_count - # maps langchain_core.messages.ai.UsageMetadata object + # maps langchain_core.messages.ai.UsageMetadata object. For some providers + # (e.g. OpenAI with stream_usage) this ends up merged into llm_output's + # token usage, but others (e.g. ChatAnthropic) only normalize cache + # details onto the generated AIMessage itself (message.usage_metadata), + # so prefer the message's own usage_metadata when present and fall back + # to the llm_output-based token_usage otherwise. + usage_metadata_sources = [source for source in (message_usage, token_usage) if source is not None] for attribute_name, details_key_or_none, keys in [ (GEN_AI_USAGE_INPUT_TOKENS_KEY, None, ("input_tokens",)), (GEN_AI_USAGE_OUTPUT_TOKENS_KEY, None, ("output_tokens",)), @@ -386,10 +406,11 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] ("reasoning",), ), ]: - details = token_usage.get(details_key_or_none) if details_key_or_none else token_usage - if details is not None: - if (token_count := get_first_value(details, keys)) is not None: + for source in usage_metadata_sources: + details = source.get(details_key_or_none) if details_key_or_none else source + if details is not None and (token_count := get_first_value(details, keys)) is not None: yield attribute_name, token_count + break def parse_token_usage_for_non_streaming_outputs( @@ -418,6 +439,50 @@ def parse_token_usage_for_non_streaming_outputs( return None +def _message_usage_metadata(outputs: Mapping[str, Any] | None) -> Mapping[str, Any] | None: + """ + Extracts the normalized `usage_metadata` (LangChain's `UsageMetadata`) + from the first generated message in `outputs["generations"]`, if present. + + LangChain attaches its normalized token/cache usage directly to the + generated `AIMessage` (`message.usage_metadata`) rather than always + surfacing it on `llm_output` - for example, ChatAnthropic's raw + `llm_output["usage"]` only has flat, non-normalized fields, while the + normalized cache details live on the message. Handles both an actual + `BaseMessage` instance (attribute access) and LangChain's `dumpd`-style + serialized dict form (`{"type": "constructor", "kwargs": {...}}`). + """ + if not outputs or not isinstance(outputs, Mapping): + return None + generations = outputs.get("generations") + if not generations or not isinstance(generations, Iterable): + return None + first_generations = next(iter(generations), None) + if not first_generations or not isinstance(first_generations, Iterable): + return None + + for generation in first_generations: + message = ( + generation.get("message") if isinstance(generation, Mapping) else getattr(generation, "message", None) + ) + if message is None: + continue + + if isinstance(message, BaseMessage): + usage_metadata = getattr(message, "usage_metadata", None) + elif isinstance(message, Mapping): + kwargs = message.get("kwargs") + usage_metadata = kwargs.get("usage_metadata") if isinstance(kwargs, Mapping) else None + if usage_metadata is None: + usage_metadata = message.get("usage_metadata") + else: + usage_metadata = None + + if usage_metadata: + return usage_metadata + return None + + @stop_on_exception def function_calls(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, str]]: """ diff --git a/tests/observability/core/test_inference_scope.py b/tests/observability/core/test_inference_scope.py index 6057cc59..88bc4af5 100644 --- a/tests/observability/core/test_inference_scope.py +++ b/tests/observability/core/test_inference_scope.py @@ -24,11 +24,11 @@ from microsoft_agents_a365.observability.core.constants import ( CHANNEL_LINK_KEY, CHANNEL_NAME_KEY, - CHAT_OPERATION_NAME, GEN_AI_AGENT_ID_KEY, GEN_AI_AGENT_TYPE_KEY, GEN_AI_OPERATION_NAME_KEY, GEN_AI_PROVIDER_NAME_KEY, + GEN_AI_REQUEST_MODEL_KEY, GEN_AI_RESPONSE_MODEL_KEY, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, @@ -37,6 +37,7 @@ SERVER_ADDRESS_KEY, SERVER_PORT_KEY, ) +from microsoft_agents_a365.observability.core.models.agent_type import AgentType from microsoft_agents_a365.observability.core.opentelemetry_scope import OpenTelemetryScope from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -155,7 +156,7 @@ def test_inference_scope_emits_model_observation_attributes(self): span = self.span_exporter.get_finished_spans()[-1] attributes = span.attributes or {} - self.assertEqual(attributes[GEN_AI_OPERATION_NAME_KEY], CHAT_OPERATION_NAME) + self.assertEqual(attributes[GEN_AI_OPERATION_NAME_KEY], "text_completion") self.assertEqual(attributes[GEN_AI_PROVIDER_NAME_KEY], "openai") self.assertEqual(attributes[GEN_AI_RESPONSE_MODEL_KEY], "gpt-4.1-2025-04-14") self.assertEqual(attributes[GEN_AI_AGENT_ID_KEY], "test-inference-agent") @@ -167,6 +168,76 @@ def test_inference_scope_emits_model_observation_attributes(self): self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 30) self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + def test_inference_scope_operation_name_mapping_per_type(self): + """Each InferenceOperationType maps to its canonical lowercase semconv value, + used for both the span name prefix and gen_ai.operation.name - not hardcoded + to 'chat' for every operation type.""" + cases = [ + (InferenceOperationType.CHAT, "chat"), + (InferenceOperationType.TEXT_COMPLETION, "text_completion"), + (InferenceOperationType.GENERATE_CONTENT, "generate_content"), + ] + for operation_type, expected_name in cases: + with self.subTest(operation_type=operation_type): + details = InferenceCallDetails( + operationName=operation_type, + model="gemini-2.0-flash", + providerName="gcp.gemini", + ) + + scope = InferenceScope.start(Request(), details, self.agent_details) + scope.dispose() + + span = self.span_exporter.get_finished_spans()[-1] + attributes = span.attributes or {} + + self.assertEqual(attributes[GEN_AI_OPERATION_NAME_KEY], expected_name) + self.assertEqual(span.name, f"{expected_name} gemini-2.0-flash") + + def test_whitespace_only_model_is_normalized_when_response_model_present(self): + """A whitespace-only `model` is normalized to None (not treated as present) + when a valid `responseModel` is also provided.""" + details = InferenceCallDetails( + operationName=InferenceOperationType.CHAT, + model=" ", + providerName="openai", + responseModel="gpt-4.1", + ) + + self.assertIsNone(details.model) + self.assertEqual(details.responseModel, "gpt-4.1") + + scope = InferenceScope.start(Request(), details, self.agent_details) + scope.dispose() + + span = self.span_exporter.get_finished_spans()[-1] + attributes = span.attributes or {} + + self.assertEqual(span.name, "chat gpt-4.1") + self.assertNotIn(GEN_AI_REQUEST_MODEL_KEY, attributes) + self.assertEqual(attributes[GEN_AI_RESPONSE_MODEL_KEY], "gpt-4.1") + + def test_agent_type_enum_member_emits_string_value(self): + """Passing an AgentType enum member for agent_type emits its string + `.value` as the span attribute, instead of being silently dropped by OTel.""" + details = InferenceCallDetails( + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", + ) + agent_details = AgentDetails( + agent_id="test-inference-agent", + agent_type=AgentType.FOUNDRY, + ) + + scope = InferenceScope.start(Request(), details, agent_details) + scope.dispose() + + span = self.span_exporter.get_finished_spans()[-1] + attributes = span.attributes or {} + + self.assertEqual(attributes[GEN_AI_AGENT_TYPE_KEY], "Foundry") + def test_inference_scope_start_method(self): """Test InferenceScope.start() static method.""" details = InferenceCallDetails( diff --git a/tests/observability/extensions/langchain/test_wrapper_langchain.py b/tests/observability/extensions/langchain/test_wrapper_langchain.py index 7ab01572..6e265270 100644 --- a/tests/observability/extensions/langchain/test_wrapper_langchain.py +++ b/tests/observability/extensions/langchain/test_wrapper_langchain.py @@ -51,6 +51,80 @@ def test_cache_token_counts_use_standard_attributes(self): self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + def test_cache_token_counts_read_from_generated_message_usage_metadata(self): + """ + Test that cache token counts are extracted from the generated + message's normalized `usage_metadata` (LangChain's `dumpd`-style + serialized AIMessage), not just from a synthetic `llm_output` + location. This mirrors the actual shape LangChain produces for + providers such as ChatAnthropic, where the raw `llm_output["usage"]` + only carries flat (non-normalized) fields while the message itself + carries the normalized `input_token_details`. + """ + outputs = { + "llm_output": { + # Anthropic's raw usage payload: flat cache fields, no + # normalized input_token_details. + "usage": { + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 40, + "cache_creation_input_tokens": 10, + } + }, + "generations": [ + [ + { + "message": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "messages", "AIMessage"], + "kwargs": { + "content": "hello", + "usage_metadata": { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_token_details": { + "cache_read": 40, + "cache_creation": 10, + }, + }, + }, + } + } + ] + ], + } + + attributes = dict(token_counts(outputs)) + + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + + def test_cache_token_counts_fall_back_to_raw_anthropic_flat_fields(self): + """ + Test that cache token counts fall back to Anthropic's raw, flat + `llm_output["usage"]` fields when no normalized `usage_metadata` is + available on the generated message. + """ + outputs = { + "llm_output": { + "usage": { + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 40, + "cache_creation_input_tokens": 10, + } + }, + "generations": [[{"message": {"lc": 1, "type": "constructor", "kwargs": {"content": "hello"}}}]], + } + + attributes = dict(token_counts(outputs)) + + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + class TestInstrumentorLangChain(unittest.TestCase): """Unit tests for InstrumentorForLangChain class.""" From f722e885269d352977b0b888416e1dbc3024259e Mon Sep 17 00:00:00 2001 From: Pujarini Mohapatra Date: Thu, 15 Oct 2026 12:28:52 +0100 Subject: [PATCH 3/4] Fix reasoning-token mislabeling and batch usage aggregation in LangChain utils - Remove erroneous mapping of output_token_details.reasoning (a token count) to GEN_AI_RESPONSE_FINISH_REASONS_KEY in token_counts(); that attribute is reserved for actual finish reasons. - Rewrite _message_usage_metadata() to aggregate usage across ALL batch entries in outputs["generations"] instead of only the first, so batched LLM calls no longer silently drop usage for later inputs. Only the first completion with usage data per batch entry is used to avoid double-counting alternative completions (n > 1). - Add _merge_usage_metadata() helper to recursively sum numeric usage fields (including nested detail dicts) across batch entries. - Add regression tests for both fixes. - Reformat utils.py and test_wrapper_langchain.py with ruff to fix CI format-check failure (ruff format --check .). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../extensions/langchain/utils.py | 104 +++++++++++++----- .../langchain/test_wrapper_langchain.py | 99 ++++++++++++++++- 2 files changed, 174 insertions(+), 29 deletions(-) diff --git a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py index 30a2941c..4e93e72a 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py +++ b/libraries/microsoft-agents-a365-observability-extensions-langchain/microsoft_agents_a365/observability/extensions/langchain/utils.py @@ -349,7 +349,10 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] ), ), ]: - if token_usage is not None and (token_count := get_first_value(token_usage, keys)) is not None: + if ( + token_usage is not None + and (token_count := get_first_value(token_usage, keys)) is not None + ): yield attribute_name, token_count if token_usage is not None: @@ -386,7 +389,9 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] # details onto the generated AIMessage itself (message.usage_metadata), # so prefer the message's own usage_metadata when present and fall back # to the llm_output-based token_usage otherwise. - usage_metadata_sources = [source for source in (message_usage, token_usage) if source is not None] + usage_metadata_sources = [ + source for source in (message_usage, token_usage) if source is not None + ] for attribute_name, details_key_or_none, keys in [ (GEN_AI_USAGE_INPUT_TOKENS_KEY, None, ("input_tokens",)), (GEN_AI_USAGE_OUTPUT_TOKENS_KEY, None, ("output_tokens",)), @@ -400,11 +405,9 @@ def token_counts(outputs: Mapping[str, Any] | None) -> Iterator[tuple[str, int]] "input_token_details", ("cache_write", "cache_creation"), ), - ( - GEN_AI_RESPONSE_FINISH_REASONS_KEY, - "output_token_details", - ("reasoning",), - ), + # Note: `output_token_details.reasoning` is intentionally not mapped + # here - it is a reasoning *token count*, not a finish reason, and + # there is no dedicated reasoning-tokens attribute defined yet. ]: for source in usage_metadata_sources: details = source.get(details_key_or_none) if details_key_or_none else source @@ -439,10 +442,30 @@ def parse_token_usage_for_non_streaming_outputs( return None +def _merge_usage_metadata(aggregate: dict[str, Any], usage_metadata: Mapping[str, Any]) -> None: + """ + Merges `usage_metadata` into `aggregate` in place, summing numeric token + counts (including nested detail mappings like `input_token_details`) so + that usage from multiple batch entries is aggregated rather than + overwritten. + """ + for key, value in usage_metadata.items(): + if isinstance(value, Mapping): + nested = aggregate.get(key) + if not isinstance(nested, dict): + nested = {} + aggregate[key] = nested + _merge_usage_metadata(nested, value) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + aggregate[key] = aggregate.get(key, 0) + value + elif key not in aggregate: + aggregate[key] = value + + def _message_usage_metadata(outputs: Mapping[str, Any] | None) -> Mapping[str, Any] | None: """ Extracts the normalized `usage_metadata` (LangChain's `UsageMetadata`) - from the first generated message in `outputs["generations"]`, if present. + from the generated messages in `outputs["generations"]`, if present. LangChain attaches its normalized token/cache usage directly to the generated `AIMessage` (`message.usage_metadata`) rather than always @@ -451,36 +474,61 @@ def _message_usage_metadata(outputs: Mapping[str, Any] | None) -> Mapping[str, A normalized cache details live on the message. Handles both an actual `BaseMessage` instance (attribute access) and LangChain's `dumpd`-style serialized dict form (`{"type": "constructor", "kwargs": {...}}`). + + `outputs["generations"]` is a list of batches, each containing one or + more alternative completions for the same input (e.g. when `n > 1`). + Alternative completions for the same input share the same usage + accounting, so only the first completion per batch entry is used; usage + is then summed across batch entries so multi-input batch calls aren't + under-reported. """ if not outputs or not isinstance(outputs, Mapping): return None generations = outputs.get("generations") if not generations or not isinstance(generations, Iterable): return None - first_generations = next(iter(generations), None) - if not first_generations or not isinstance(first_generations, Iterable): - return None - for generation in first_generations: - message = ( - generation.get("message") if isinstance(generation, Mapping) else getattr(generation, "message", None) - ) - if message is None: + aggregate: dict[str, Any] | None = None + for batch_entry in generations: + if not batch_entry or not isinstance(batch_entry, Iterable): continue - if isinstance(message, BaseMessage): - usage_metadata = getattr(message, "usage_metadata", None) - elif isinstance(message, Mapping): - kwargs = message.get("kwargs") - usage_metadata = kwargs.get("usage_metadata") if isinstance(kwargs, Mapping) else None - if usage_metadata is None: - usage_metadata = message.get("usage_metadata") - else: - usage_metadata = None + usage_metadata = None + for generation in batch_entry: + message = ( + generation.get("message") + if isinstance(generation, Mapping) + else getattr(generation, "message", None) + ) + if message is None: + continue + + if isinstance(message, BaseMessage): + usage_metadata = getattr(message, "usage_metadata", None) + elif isinstance(message, Mapping): + kwargs = message.get("kwargs") + usage_metadata = ( + kwargs.get("usage_metadata") if isinstance(kwargs, Mapping) else None + ) + if usage_metadata is None: + usage_metadata = message.get("usage_metadata") + else: + usage_metadata = None - if usage_metadata: - return usage_metadata - return None + if usage_metadata: + # Only the first completion with usage data in this batch + # entry is used - alternative completions (n > 1) for the + # same input would otherwise be double-counted. + break + + if not usage_metadata: + continue + + if aggregate is None: + aggregate = {} + _merge_usage_metadata(aggregate, usage_metadata) + + return aggregate @stop_on_exception diff --git a/tests/observability/extensions/langchain/test_wrapper_langchain.py b/tests/observability/extensions/langchain/test_wrapper_langchain.py index 6e265270..ce93b7da 100644 --- a/tests/observability/extensions/langchain/test_wrapper_langchain.py +++ b/tests/observability/extensions/langchain/test_wrapper_langchain.py @@ -8,9 +8,12 @@ from microsoft_agents_a365.observability.core import configure from microsoft_agents_a365.observability.core.constants import ( GEN_AI_REQUEST_MODEL_KEY, + GEN_AI_RESPONSE_FINISH_REASONS_KEY, GEN_AI_RESPONSE_MODEL_KEY, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_INPUT_TOKENS_KEY, + GEN_AI_USAGE_OUTPUT_TOKENS_KEY, ) from microsoft_agents_a365.observability.extensions.langchain.tracer_instrumentor import ( CustomLangChainInstrumentor, @@ -117,7 +120,9 @@ def test_cache_token_counts_fall_back_to_raw_anthropic_flat_fields(self): "cache_creation_input_tokens": 10, } }, - "generations": [[{"message": {"lc": 1, "type": "constructor", "kwargs": {"content": "hello"}}}]], + "generations": [ + [{"message": {"lc": 1, "type": "constructor", "kwargs": {"content": "hello"}}}] + ], } attributes = dict(token_counts(outputs)) @@ -125,6 +130,98 @@ def test_cache_token_counts_fall_back_to_raw_anthropic_flat_fields(self): self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 40) self.assertEqual(attributes[GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY], 10) + def test_reasoning_tokens_from_message_usage_metadata_are_not_emitted_as_finish_reasons(self): + """ + Test that a reasoning *token count* sourced only from the generated + message's normalized `usage_metadata` (no `llm_output` present) is + never emitted under `GEN_AI_RESPONSE_FINISH_REASONS_KEY` - that + attribute is reserved for actual finish reasons, not token counts. + """ + outputs = { + "generations": [ + [ + { + "message": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "messages", "AIMessage"], + "kwargs": { + "content": "hello", + "usage_metadata": { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "output_token_details": { + "reasoning": 8, + }, + }, + }, + } + } + ] + ], + } + + attributes = dict(token_counts(outputs)) + + self.assertNotIn(GEN_AI_RESPONSE_FINISH_REASONS_KEY, attributes) + self.assertEqual(attributes[GEN_AI_USAGE_INPUT_TOKENS_KEY], 100) + self.assertEqual(attributes[GEN_AI_USAGE_OUTPUT_TOKENS_KEY], 20) + + def test_token_counts_are_aggregated_across_batch_entries(self): + """ + Test that `token_counts` sums usage across every batch entry in + `outputs["generations"]` (one entry per input in a batched call), + rather than only reading the first batch entry and silently + dropping usage for the rest. + """ + outputs = { + "generations": [ + [ + { + "message": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "messages", "AIMessage"], + "kwargs": { + "content": "first", + "usage_metadata": { + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "input_token_details": {"cache_read": 1}, + }, + }, + } + } + ], + [ + { + "message": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "messages", "AIMessage"], + "kwargs": { + "content": "second", + "usage_metadata": { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_token_details": {"cache_read": 4}, + }, + }, + } + } + ], + ], + } + + attributes = dict(token_counts(outputs)) + + self.assertEqual(attributes[GEN_AI_USAGE_INPUT_TOKENS_KEY], 110) + self.assertEqual(attributes[GEN_AI_USAGE_OUTPUT_TOKENS_KEY], 22) + self.assertEqual(attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY], 5) + class TestInstrumentorLangChain(unittest.TestCase): """Unit tests for InstrumentorForLangChain class.""" From ca21032e9123a10b611b6c80b042323b7ecb992e Mon Sep 17 00:00:00 2001 From: Pujarini Mohapatra Date: Mon, 19 Oct 2026 12:08:21 +0100 Subject: [PATCH 4/4] Document model observation migration and usage semantics Address review feedback on canonical operation and span-name changes, model identity validation, request/response model separation, and normalized cache-inclusive input-token totals. Update the existing OpenTelemetry integration guide without changing runtime behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...integrating-with-existing-opentelemetry.md | 15 ++++++++--- .../CHANGELOG.md | 21 ++++++++++----- .../CHANGELOG.md | 26 +++++++++++++------ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/docs/integrating-with-existing-opentelemetry.md b/docs/integrating-with-existing-opentelemetry.md index e72498a3..bd5bb970 100644 --- a/docs/integrating-with-existing-opentelemetry.md +++ b/docs/integrating-with-existing-opentelemetry.md @@ -93,10 +93,15 @@ The SDK produces three core span kinds. Your backend should show them in this ty | `gen_ai.operation.name` | Produced by | Typical parent | Span name (default) | Notes | |-------------------------|---------------------------------------------------|----------------|---------------------|-------| | `invoke_agent` | `InvokeAgentScope` (one per user turn) | (root or app) | `invoke_agent ` when set, else `invoke_agent` | | -| (varies — see notes) | `InferenceScope` (one per LLM call) | `invoke_agent` | ` ` | **Manual instrumentation** uses `InferenceOperationType.value` (currently `Chat` / `TextCompletion` / `GenerateContent`, capitalized). **Auto-instrumentation** (e.g. `OpenAIAgentsTraceInstrumentor`) uses lowercase per the [OTel GenAI semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/) (e.g. `chat`). The two are inconsistent today. | +| `chat` / `text_completion` / `generate_content` | `InferenceScope` (one per LLM call) | `invoke_agent` | ` ` | Manual inference scopes use canonical lowercase [OTel GenAI operation names](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/), matching auto-instrumentation for the corresponding operation. | | `execute_tool` | `ExecuteToolScope` (one per tool invocation) | `invoke_agent` | `execute_tool ` (always includes the tool name) | Records tool name, args, and result. | -Filter your backend by the `gen_ai.operation.name` attribute or by span name. Note that `inference` is *not* the literal attribute value — manual instrumentation produces `Chat` / `TextCompletion` / `GenerateContent` (the `InferenceOperationType.value`), while auto-instrumentation extension packages produce the lowercase OTel-spec form (e.g. `chat`). This casing discrepancy is tracked as an SDK issue. +Filter your backend by the `gen_ai.operation.name` attribute or by span name. +`InferenceScope` emits `chat`, `text_completion`, or `generate_content` according +to the selected `InferenceOperationType`, with span names such as `chat gpt-4`. +`inference` is not a literal operation value. When upgrading, update case-sensitive +filters that matched the previous `Chat`, `TextCompletion`, or `GenerateContent` +values and span-name prefixes, or compare case-insensitively. ## Verifying the integration @@ -110,7 +115,11 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExport trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) ``` -Run a single turn. If you see `invoke_agent` / `Chat` (or `chat`) / `execute_tool` JSON dumps on stdout, the SDK is producing spans correctly — the issue is in your backend exporter (network, auth, sampling). If you don't see them, the integration itself is wrong; check the pitfalls below. +Run a single turn. If you see `invoke_agent`, the inference operation +(`chat`, `text_completion`, or `generate_content`), and any invoked `execute_tool` +spans as JSON dumps on stdout, the SDK is producing spans correctly - investigate +your backend exporter (network, auth, sampling). If expected spans are missing +from stdout too, check the integration pitfalls below. ## Common pitfalls diff --git a/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md b/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md index 092a6274..af793b31 100644 --- a/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md +++ b/libraries/microsoft-agents-a365-observability-core/CHANGELOG.md @@ -6,12 +6,6 @@ All notable changes to this package will be documented in this file. ### Fixed -- **`InferenceScope` span name and `gen_ai.operation.name` now reflect the actual operation** — - Previously every inference span was hardcoded to `chat`, regardless of - `InferenceCallDetails.operationName`. The span name and the - `gen_ai.operation.name` attribute are now derived from the canonical - lowercase semconv value of the operation (e.g. `chat`, `text_completion`, - `generate_content`), matching the pre-existing Node.js/.NET behavior. - **Agent 365 exporter's `GEN_AI_OPERATION_NAMES` allowlist now recognizes the canonical lowercase operation names** (`chat`, `text_completion`, `generate_content`) in addition to the existing aliases, since spans can now @@ -30,6 +24,16 @@ All notable changes to this package will be documented in this file. ### Breaking Changes +- **Manual inference operation names and span-name prefixes now use canonical + OpenTelemetry values** — `Chat`, `TextCompletion`, and `GenerateContent` + become `chat`, `text_completion`, and `generate_content`, respectively. + For example, `Chat gpt-4` becomes `chat gpt-4`. Text-completion and + generate-content spans that were previously filtered out are now exported + to Agent 365. +- **`InferenceCallDetails` now validates model identity during construction** — + If both `model` and `responseModel` are missing, empty, or whitespace-only, + construction raises `ValueError`. Previously these values were accepted + without validation. This happens before an `InferenceScope` is created. - **OBS exports always use `/observabilityService`** — The `use_s2s_endpoint` option is deprecated and ignored, even when `False`. Exports no longer select or fall back to `/observability`. Provide an app-only OBS token independently @@ -44,6 +48,11 @@ All notable changes to this package will be documented in this file. ### Migration +- Update case-sensitive operation-name and span-name filters, dashboards, and + alerts to the canonical values above, or compare case-insensitively. +- Supply a non-blank `model` or `responseModel` when constructing + `InferenceCallDetails`; validate optional configuration before constructing + the telemetry object so missing model identity does not interrupt a request. - Request the OBS resource `/.default` scope (`api://9b975845-388f-4429-889e-eab1ef63949c/.default`) for the exporting agent identity. Do not use delegated `Agent365.Observability.OtelWrite` tokens diff --git a/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md b/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md index f58dc5aa..6c0ae5b9 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md +++ b/libraries/microsoft-agents-a365-observability-extensions-langchain/CHANGELOG.md @@ -4,16 +4,26 @@ All notable changes to this package will be documented in this file. ## [Unreleased] -### Fixed +### Added -- **Cache token counts now read the generated message's normalized - `usage_metadata`** (`gen_ai.usage.cache_read_input_tokens` / - `gen_ai.usage.cache_write_input_tokens`), falling back to the - provider-specific `llm_output` usage payload. Previously only - `llm_output.token_usage`/`llm_output.usage` were checked, which missed - providers (e.g. ChatAnthropic) that normalize cache details onto the - generated `AIMessage` instead. +- **Cache token attributes** are now captured from the generated message's normalized + `usage_metadata` (`gen_ai.usage.cache_read.input_tokens` / + `gen_ai.usage.cache_write.input_tokens`) or provider-specific `llm_output` + usage payloads. The released package did not emit cache-token attributes. ### Changed +- **Request and response model identity are now separate** - the served model + from `llm_output.model_name` or `llm_output.model` moves from + `gen_ai.request.model` to `gen_ai.response.model`. The request model now + comes from `metadata.ls_model_name` or `invocation_params`; it remains absent + when neither source supplies it. Update dashboards that previously treated + the served model as the request model. +- **Input-token counts use normalized message usage when available** - for + ChatAnthropic prompt caching, `gen_ai.usage.input_tokens` includes uncached + input, cache reads, and cache creation rather than only the raw Anthropic + input count. For example, 100 uncached + 40 cache-read + 10 cache-creation + tokens reports 150 instead of 100. Do not add cache-token counts to this + normalized total again. Batched generations retain aggregate `llm_output` + usage rather than replacing it with a single message's usage. - Removed `set_execution_type()` function and all `ExecutionType` / `GEN_AI_EXECUTION_TYPE_KEY` usage.