From c65f82d1945679296f948ba0355d4398a830bccc Mon Sep 17 00:00:00 2001 From: Leroyyyyyyyyy <150530443+Leroyyyyyyyyy@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:01:58 +0800 Subject: [PATCH 1/5] fix(otel): share one AnyValue decoder across all OTLP paths Three AnyValue decoders had drifted apart. extraction's flatten_otlp_attributes silently dropped array/kvlist/bytes attributes, and the OTLP JSON loader json.dumps()'d the raw proto wrapper instead of decoding it. gen_ai.response.finish_reasons therefore surfaced as '{"values": [{"stringValue": "stop"}]}' on one path and vanished on another, where both should yield ["stop"]. Move the already-correct recursive decoder out of api/otlp_processing.py into a dependency-free module and route all three call sites through it. The new module imports only the standard library, so extraction, loader.otlp and api.otlp_processing can all share it without creating an import cycle. bytesValue is returned unchanged: MessageToDict base64-encodes protobuf bytes fields, so callers already receive a str today. Decoding it here would change existing behaviour, which is out of scope for this fix. Adds coverage for array/kvlist/bytes attributes, which previously had none on either path. Fixes #173 --- src/agentevals/api/otlp_processing.py | 30 +--------- src/agentevals/extraction.py | 21 +++---- src/agentevals/loader/otlp.py | 21 +------ src/agentevals/otlp_anyvalue.py | 80 +++++++++++++++++++++++++++ tests/test_extraction.py | 52 +++++++++++++++++ tests/test_otlp_loader.py | 60 ++++++++++++++++++++ 6 files changed, 204 insertions(+), 60 deletions(-) create mode 100644 src/agentevals/otlp_anyvalue.py diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 948e31b..613510e 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -15,6 +15,7 @@ ) from ..extraction import flatten_otlp_attributes +from ..otlp_anyvalue import decode_any_value from ..trace_attrs import ( OTEL_GENAI_CONVERSATION_ID, OTEL_GENAI_INPUT_MESSAGES, @@ -310,37 +311,12 @@ def _convert_otlp_log_record(log_record: dict) -> dict | None: return result -def _parse_otlp_any_value(value_obj: dict): - """Recursively parse an OTLP AnyValue to native Python types. - - Handles the full AnyValue union: stringValue, intValue, doubleValue, - boolValue, kvlistValue (→ dict), arrayValue (→ list), bytesValue. - """ - if "stringValue" in value_obj: - return value_obj["stringValue"] - if "intValue" in value_obj: - return int(value_obj["intValue"]) - if "doubleValue" in value_obj: - return float(value_obj["doubleValue"]) - if "boolValue" in value_obj: - return value_obj["boolValue"] - if "kvlistValue" in value_obj: - kv = value_obj["kvlistValue"] - return {item.get("key", ""): _parse_otlp_any_value(item.get("value", {})) for item in kv.get("values", [])} - if "arrayValue" in value_obj: - arr = value_obj["arrayValue"] - return [_parse_otlp_any_value(v) for v in arr.get("values", [])] - if "bytesValue" in value_obj: - return value_obj["bytesValue"] - return value_obj - - def _parse_otlp_body(body_raw: dict) -> dict | str: """Parse OTLP log record body value. Top-level stringValue bodies are JSON-decoded (Strands-style logs store message content as JSON strings). All other AnyValue types are parsed - recursively via ``_parse_otlp_any_value`` (handles the nested kvlistValue / + recursively via ``decode_any_value`` (handles the nested kvlistValue / arrayValue structures used by the OpenAI instrumentor). """ if "stringValue" in body_raw: @@ -351,4 +327,4 @@ def _parse_otlp_body(body_raw: dict) -> dict | str: return json.loads(raw) except (json.JSONDecodeError, TypeError): return raw - return _parse_otlp_any_value(body_raw) + return decode_any_value(body_raw) diff --git a/src/agentevals/extraction.py b/src/agentevals/extraction.py index 141f230..efcc9d6 100644 --- a/src/agentevals/extraction.py +++ b/src/agentevals/extraction.py @@ -17,6 +17,7 @@ from typing import Any, Protocol, TypedDict, TypeVar from .loader.base import Span, Trace +from .otlp_anyvalue import decode_attributes from .trace_attrs import ( ADK_LLM_REQUEST, ADK_LLM_RESPONSE, @@ -523,20 +524,12 @@ def is_invocation_span(span: Span) -> bool: def flatten_otlp_attributes(attrs_list: list[dict]) -> dict[str, Any]: - """Convert OTLP attributes array [{key, value: {stringValue|...}}] to flat dict.""" - result: dict[str, Any] = {} - for attr in attrs_list: - key = attr.get("key", "") - value_obj = attr.get("value", {}) - if "stringValue" in value_obj: - result[key] = value_obj["stringValue"] - elif "intValue" in value_obj: - result[key] = int(value_obj["intValue"]) - elif "doubleValue" in value_obj: - result[key] = float(value_obj["doubleValue"]) - elif "boolValue" in value_obj: - result[key] = value_obj["boolValue"] - return result + """Convert OTLP attributes array [{key, value: {stringValue|...}}] to flat dict. + + Delegates to the shared ``AnyValue`` decoder so array/kvlist/bytes + attributes survive instead of being dropped. + """ + return decode_attributes(attrs_list) # --------------------------------------------------------------------------- diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index ef26cb2..26a11ea 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -5,6 +5,7 @@ import json import logging +from ..otlp_anyvalue import decode_attributes from ..trace_attrs import ( OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, @@ -192,25 +193,7 @@ def _extract_attributes(self, attrs) -> dict: if isinstance(attrs, dict): return self._flatten_nested_dict(attrs) - result = {} - for attr in attrs: - key = attr.get("key", "") - value_obj = attr.get("value", {}) - - if "stringValue" in value_obj: - result[key] = value_obj["stringValue"] - elif "intValue" in value_obj: - result[key] = int(value_obj["intValue"]) - elif "doubleValue" in value_obj: - result[key] = float(value_obj["doubleValue"]) - elif "boolValue" in value_obj: - result[key] = value_obj["boolValue"] - elif "arrayValue" in value_obj: - result[key] = json.dumps(value_obj["arrayValue"]) - elif "kvlistValue" in value_obj: - result[key] = json.dumps(value_obj["kvlistValue"]) - - return result + return decode_attributes(attrs) @staticmethod def _flatten_nested_dict(d: dict, prefix: str = "") -> dict: diff --git a/src/agentevals/otlp_anyvalue.py b/src/agentevals/otlp_anyvalue.py new file mode 100644 index 0000000..9f498d3 --- /dev/null +++ b/src/agentevals/otlp_anyvalue.py @@ -0,0 +1,80 @@ +"""Shared decoder for the OTLP ``AnyValue`` union. + +OTLP encodes every attribute value, log body and nested element as an +``AnyValue``: a one-of wrapper such as ``{"stringValue": "chat"}`` or +``{"arrayValue": {"values": [...]}}``. The protobuf receiver (via +``MessageToDict``) and OTLP/JSON payloads deliver that same dict shape, so +every consumer needs identical decoding rules. + +This module is deliberately dependency-free: importing only the standard +library lets ``extraction``, ``loader.otlp`` and ``api.otlp_processing`` all +use it without creating an import cycle. +""" + +from __future__ import annotations + +from typing import Any + +ANY_VALUE_FIELDS = ( + "stringValue", + "intValue", + "doubleValue", + "boolValue", + "kvlistValue", + "arrayValue", + "bytesValue", +) + + +def decode_any_value(value_obj: dict) -> Any: + """Recursively decode an OTLP ``AnyValue`` to a native Python value. + + Handles the full union: stringValue, intValue (OTLP sends it as a + string), doubleValue, boolValue, kvlistValue (→ dict), arrayValue + (→ list), bytesValue. + + ``bytesValue`` is returned unchanged. ``MessageToDict`` base64-encodes + protobuf bytes fields and OTLP/JSON does the same, so callers already + receive a str; decoding it here would change the value they see today. + + A value carrying none of the union fields is returned as-is. + """ + if "stringValue" in value_obj: + return value_obj["stringValue"] + if "intValue" in value_obj: + return int(value_obj["intValue"]) + if "doubleValue" in value_obj: + return float(value_obj["doubleValue"]) + if "boolValue" in value_obj: + return value_obj["boolValue"] + if "kvlistValue" in value_obj: + kv = value_obj["kvlistValue"] + return {item.get("key", ""): decode_any_value(item.get("value", {})) for item in kv.get("values", [])} + if "arrayValue" in value_obj: + arr = value_obj["arrayValue"] + return [decode_any_value(v) for v in arr.get("values", [])] + if "bytesValue" in value_obj: + return value_obj["bytesValue"] + return value_obj + + +def is_any_value(value_obj: dict) -> bool: + """Return True when *value_obj* carries one of the ``AnyValue`` fields.""" + for field in ANY_VALUE_FIELDS: + if field in value_obj: + return True + return False + + +def decode_attributes(attrs_list: list[dict]) -> dict[str, Any]: + """Decode an OTLP attributes array to a flat ``{key: value}`` dict. + + Entries whose value carries no ``AnyValue`` field are skipped, matching + the behaviour every call site had before they shared this decoder. + """ + result: dict[str, Any] = {} + for attr in attrs_list: + value_obj = attr.get("value", {}) + if is_any_value(value_obj): + result[attr.get("key", "")] = decode_any_value(value_obj) + return result diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 96544f7..ec3591f 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -400,6 +400,58 @@ def test_mixed_types(self): ) assert result == {"str": "hello", "num": 3.14, "flag": True} + def test_array_value(self): + result = flatten_otlp_attributes( + [ + { + "key": "gen_ai.response.finish_reasons", + "value": {"arrayValue": {"values": [{"stringValue": "stop"}]}}, + }, + ] + ) + assert result == {"gen_ai.response.finish_reasons": ["stop"]} + + def test_kvlist_value(self): + result = flatten_otlp_attributes( + [ + { + "key": "gen_ai.request.params", + "value": { + "kvlistValue": { + "values": [ + {"key": "temperature", "value": {"doubleValue": 0.7}}, + {"key": "stream", "value": {"boolValue": False}}, + ] + } + }, + }, + ] + ) + assert result == {"gen_ai.request.params": {"temperature": 0.7, "stream": False}} + + def test_array_of_kvlist(self): + """Tool calls arrive as an arrayValue of kvlistValue.""" + result = flatten_otlp_attributes( + [ + { + "key": "gen_ai.tool.calls", + "value": { + "arrayValue": { + "values": [ + {"kvlistValue": {"values": [{"key": "name", "value": {"stringValue": "get_weather"}}]}}, + ] + } + }, + }, + ] + ) + assert result == {"gen_ai.tool.calls": [{"name": "get_weather"}]} + + def test_bytes_value(self): + """MessageToDict base64-encodes bytes fields, so the decoder sees a str.""" + result = flatten_otlp_attributes([{"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}]) + assert result == {"payload": "AP9oaQ=="} + def test_empty(self): assert flatten_otlp_attributes([]) == {} diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index c3a3428..402ff25 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -258,6 +258,66 @@ def test_load_from_dict_empty_resource_spans(self): assert traces == [] +class TestAnyValueAttributes: + """Attributes carrying the full OTLP AnyValue union (array / kvlist / bytes).""" + + @staticmethod + def _load_span_with(attribute): + loader = OtlpJsonLoader() + data = { + "resourceSpans": [ + { + "resource": {"attributes": []}, + "scopeSpans": [ + { + "scope": {"name": "test-scope"}, + "spans": [ + { + "traceId": "t1", + "spanId": "s1", + "name": "test", + "startTimeUnixNano": "1000000000", + "endTimeUnixNano": "2000000000", + "attributes": [attribute], + } + ], + } + ], + } + ], + } + return loader.load_from_dict(data)[0].all_spans[0] + + def test_array_value(self): + span = self._load_span_with( + { + "key": "gen_ai.response.finish_reasons", + "value": {"arrayValue": {"values": [{"stringValue": "stop"}]}}, + } + ) + assert span.tags["gen_ai.response.finish_reasons"] == ["stop"] + + def test_kvlist_value(self): + span = self._load_span_with( + { + "key": "gen_ai.request.params", + "value": { + "kvlistValue": { + "values": [ + {"key": "temperature", "value": {"doubleValue": 0.7}}, + {"key": "stream", "value": {"boolValue": False}}, + ] + } + }, + } + ) + assert span.tags["gen_ai.request.params"] == {"temperature": 0.7, "stream": False} + + def test_bytes_value(self): + span = self._load_span_with({"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}) + assert span.tags["payload"] == "AP9oaQ==" + + class TestFlatDictAttributes: """Tests for flat dict attribute format (e.g. from simplified producers).""" From 9efc842c7d10e66b763530ba8d0c140395829dae Mon Sep 17 00:00:00 2001 From: Leroyyyyyyyyy <150530443+Leroyyyyyyyyy@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:48:59 +0800 Subject: [PATCH 2/5] fix(otel): decode AnyValue when promoting span event attributes Strands stores gen_ai.input.messages in span events, and newer GenAI semconv makes messages a complex array, so the stringValue-only promotion in the OTLP loader silently dropped anything that was not a plain string. Route it through the shared decoder like the other paths. --- src/agentevals/loader/otlp.py | 6 ++-- tests/test_otlp_loader.py | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index 26a11ea..24a4378 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -5,7 +5,7 @@ import json import logging -from ..otlp_anyvalue import decode_attributes +from ..otlp_anyvalue import decode_any_value, decode_attributes, is_any_value from ..trace_attrs import ( OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, @@ -176,8 +176,8 @@ def _promote_genai_event_attributes(self, span_data: dict, attributes: dict) -> key = attr.get("key", "") if key in self._GENAI_EVENT_KEYS and key not in attributes: value_obj = attr.get("value", {}) - if "stringValue" in value_obj: - attributes[key] = value_obj["stringValue"] + if is_any_value(value_obj): + attributes[key] = decode_any_value(value_obj) def _extract_attributes(self, attrs) -> dict: """Convert attributes to a flat ``{key: value}`` dict. diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index 402ff25..031c51e 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -317,6 +317,73 @@ def test_bytes_value(self): span = self._load_span_with({"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}) assert span.tags["payload"] == "AP9oaQ==" + @staticmethod + def _load_span_with_event_attribute(attribute): + """Span carrying a GenAI event attribute in OTLP array format.""" + loader = OtlpJsonLoader() + data = { + "resourceSpans": [ + { + "resource": {"attributes": []}, + "scopeSpans": [ + { + "scope": {"name": "test-scope"}, + "spans": [ + { + "traceId": "t1", + "spanId": "s1", + "name": "chat", + "startTimeUnixNano": "0", + "endTimeUnixNano": "0", + "attributes": [], + "events": [ + { + "timeUnixNano": "0", + "name": "gen_ai.client.inference.operation.details", + "attributes": [attribute], + } + ], + } + ], + } + ], + } + ], + } + return loader.load_from_dict(data)[0].all_spans[0] + + def test_event_promotion_decodes_array_value(self): + """Strands stores messages in span events, and newer GenAI semconv makes + them a complex array. Promotion must decode it instead of dropping it.""" + span = self._load_span_with_event_attribute( + { + "key": "gen_ai.input.messages", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + {"key": "role", "value": {"stringValue": "user"}}, + {"key": "content", "value": {"stringValue": "Hello"}}, + ] + } + } + ] + } + }, + } + ) + assert span.tags["gen_ai.input.messages"] == [{"role": "user", "content": "Hello"}] + + def test_event_promotion_keeps_string_value(self): + """The pre-existing stringValue path must keep working unchanged.""" + messages_json = '[{"role": "user", "content": "Hello"}]' + span = self._load_span_with_event_attribute( + {"key": "gen_ai.output.messages", "value": {"stringValue": messages_json}} + ) + assert span.tags["gen_ai.output.messages"] == messages_json + class TestFlatDictAttributes: """Tests for flat dict attribute format (e.g. from simplified producers).""" From 3029ef7ee4cddb068d5535251a46db0cea9b335d Mon Sep 17 00:00:00 2001 From: Leroyyyyyyyyy <150530443+Leroyyyyyyyyy@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:48:27 +0800 Subject: [PATCH 3/5] fix(otel): read session_name and eval_set_id as strings only flatten_otlp_attributes used to guarantee a scalar or nothing. Now that it delegates to the shared AnyValue decoder it can also return lists and dicts, which are unhashable and raise TypeError where these values are used as dict keys (_active_session_for_name in ws_server.py and otlp_processing.py). Add a string-only accessor and use it for both fields, matching what _extract_conversation_id already did; that function now shares the same implementation. The decoded value stays available in resource_attrs. --- src/agentevals/api/otlp_processing.py | 31 ++++++++++++++++----- tests/test_otlp_receiver.py | 40 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 613510e..0230095 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -242,12 +242,32 @@ def _normalize_span(span_data: dict, scope_name: str, scope_version: str, schema return span +def _extract_string_attribute(attrs_list: list[dict], key: str) -> str | None: + """Read one OTLP attribute, accepting ``stringValue`` only. + + ``flatten_otlp_attributes`` used to guarantee a scalar or nothing. Now that + it goes through the shared ``AnyValue`` decoder it can also yield lists and + dicts, which are unhashable and would raise ``TypeError`` in the callers + that use these values as dict keys. Reading ``stringValue`` directly keeps + non-string values out instead of letting them reach that far. + """ + for attr in attrs_list: + if attr.get("key") == key: + return attr.get("value", {}).get("stringValue") + return None + + def _extract_agentevals_metadata(resource_attrs: list[dict]) -> dict: - """Extract agentevals-specific metadata from OTLP resource attributes.""" + """Extract agentevals-specific metadata from OTLP resource attributes. + + ``eval_set_id`` and ``session_name`` are read with the string-only accessor + because both are used as dict keys downstream (``_active_session_for_name``) + or typed ``str | None`` on the session models. + """ flat = flatten_otlp_attributes(resource_attrs) return { - "eval_set_id": flat.get(AGENTEVALS_EVAL_SET_ID), - "session_name": flat.get(AGENTEVALS_SESSION_NAME), + "eval_set_id": _extract_string_attribute(resource_attrs, AGENTEVALS_EVAL_SET_ID), + "session_name": _extract_string_attribute(resource_attrs, AGENTEVALS_SESSION_NAME), "service_name": flat.get("service.name"), "resource_attrs": flat, } @@ -270,10 +290,7 @@ def _prescan_conversation_id(resource_span: dict) -> str | None: def _extract_conversation_id(attrs_list: list[dict]) -> str | None: """Extract gen_ai.conversation.id from OTLP span attributes.""" - for attr in attrs_list: - if attr.get("key") == OTEL_GENAI_CONVERSATION_ID: - return attr.get("value", {}).get("stringValue") - return None + return _extract_string_attribute(attrs_list, OTEL_GENAI_CONVERSATION_ID) def _convert_otlp_log_record(log_record: dict) -> dict | None: diff --git a/tests/test_otlp_receiver.py b/tests/test_otlp_receiver.py index 629e9dc..6ec0f70 100644 --- a/tests/test_otlp_receiver.py +++ b/tests/test_otlp_receiver.py @@ -233,6 +233,46 @@ def test_missing_keys_are_none(self): assert meta["session_name"] is None assert meta["service_name"] is None + def test_non_string_session_name_is_ignored(self): + """session_name is used as a dict key in _active_session_for_name. + + The shared AnyValue decoder can now return lists and dicts, which are + unhashable; reading stringValue only keeps them out of the key path. + """ + attrs = [ + { + "key": "agentevals.session_name", + "value": {"arrayValue": {"values": [{"stringValue": "run-42"}]}}, + } + ] + meta = _extract_agentevals_metadata(attrs) + assert meta["session_name"] is None + # Would raise TypeError: unhashable type if a list leaked through. + assert {}.get(meta["session_name"]) is None + + def test_non_string_eval_set_id_is_ignored(self): + """eval_set_id is typed ``str | None`` on the session models.""" + attrs = [ + { + "key": "agentevals.eval_set_id", + "value": {"kvlistValue": {"values": [{"key": "id", "value": {"stringValue": "my-eval"}}]}}, + } + ] + meta = _extract_agentevals_metadata(attrs) + assert meta["eval_set_id"] is None + + def test_non_string_values_still_reach_resource_attrs(self): + """Narrowing applies to the two key-like fields only; the decoded value + is still available in resource_attrs for anything that wants it.""" + attrs = [ + { + "key": "agentevals.session_name", + "value": {"arrayValue": {"values": [{"stringValue": "run-42"}]}}, + } + ] + meta = _extract_agentevals_metadata(attrs) + assert meta["resource_attrs"]["agentevals.session_name"] == ["run-42"] + # --------------------------------------------------------------------------- # OTLP log record conversion From 450fe72a713ee5041c5440f55cf072c8a6c39e79 Mon Sep 17 00:00:00 2001 From: Leroyyyyyyyyy <150530443+Leroyyyyyyyyy@users.noreply.github.com> Date: Mon, 31 Aug 2026 14:23:07 +0800 Subject: [PATCH 4/5] fix(otel): narrow spec-string attributes once at the decode boundary The per-field accessor only covered the two dict-key sites we had found; flatten_otlp_attributes still handed containers to every other consumer via span.tags, so the same unhashable hazard survived elsewhere. Move the guard into decode_attributes, keyed by SPEC_STRING_ATTRS in trace_attrs. Attributes the spec types as strings come back str-or-absent; a container in one of those slots is dropped with a warning rather than JSON-dumped, since dumping would put the literal blob back into user-visible output. Attributes the spec types as arrays or structured values keep their decoded containers, which is what #173 fixes. The registry is hand-maintained: opentelemetry-semantic-conventions exposes constants and prose docstrings only, so attribute value types are not machine-readable and the set cannot be derived from the package. _extract_string_attribute is gone and _extract_conversation_id is back to its original form. Tests now assert through extract_extended_model_info_from_attrs and extract_user_text_from_attrs, the layers #173's symptom actually names, instead of stopping at the decoder and span.tags. --- src/agentevals/api/otlp_processing.py | 36 +++++++------------ src/agentevals/otlp_anyvalue.py | 36 ++++++++++++++++--- src/agentevals/trace_attrs.py | 44 +++++++++++++++++++++++ tests/test_extraction.py | 52 +++++++++++++++++++++++++++ tests/test_otlp_loader.py | 29 +++++++++++++++ tests/test_otlp_receiver.py | 22 +++++++++--- 6 files changed, 186 insertions(+), 33 deletions(-) diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 0230095..297728a 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -17,6 +17,8 @@ from ..extraction import flatten_otlp_attributes from ..otlp_anyvalue import decode_any_value from ..trace_attrs import ( + AGENTEVALS_EVAL_SET_ID, + AGENTEVALS_SESSION_NAME, OTEL_GENAI_CONVERSATION_ID, OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, @@ -31,9 +33,6 @@ logger = logging.getLogger(__name__) -AGENTEVALS_EVAL_SET_ID = "agentevals.eval_set_id" -AGENTEVALS_SESSION_NAME = "agentevals.session_name" - async def process_traces(body: dict, manager: StreamingTraceManager) -> None: """Parse ExportTraceServiceRequest and feed spans to the pipeline.""" @@ -242,32 +241,18 @@ def _normalize_span(span_data: dict, scope_name: str, scope_version: str, schema return span -def _extract_string_attribute(attrs_list: list[dict], key: str) -> str | None: - """Read one OTLP attribute, accepting ``stringValue`` only. - - ``flatten_otlp_attributes`` used to guarantee a scalar or nothing. Now that - it goes through the shared ``AnyValue`` decoder it can also yield lists and - dicts, which are unhashable and would raise ``TypeError`` in the callers - that use these values as dict keys. Reading ``stringValue`` directly keeps - non-string values out instead of letting them reach that far. - """ - for attr in attrs_list: - if attr.get("key") == key: - return attr.get("value", {}).get("stringValue") - return None - - def _extract_agentevals_metadata(resource_attrs: list[dict]) -> dict: """Extract agentevals-specific metadata from OTLP resource attributes. - ``eval_set_id`` and ``session_name`` are read with the string-only accessor - because both are used as dict keys downstream (``_active_session_for_name``) - or typed ``str | None`` on the session models. + ``eval_set_id`` and ``session_name`` are used as dict keys downstream + (``_active_session_for_name``) and typed ``str | None`` on the session + models. Both keys are in ``SPEC_STRING_ATTRS``, so the shared decoder has + already narrowed them to str-or-absent by the time they get here. """ flat = flatten_otlp_attributes(resource_attrs) return { - "eval_set_id": _extract_string_attribute(resource_attrs, AGENTEVALS_EVAL_SET_ID), - "session_name": _extract_string_attribute(resource_attrs, AGENTEVALS_SESSION_NAME), + "eval_set_id": flat.get(AGENTEVALS_EVAL_SET_ID), + "session_name": flat.get(AGENTEVALS_SESSION_NAME), "service_name": flat.get("service.name"), "resource_attrs": flat, } @@ -290,7 +275,10 @@ def _prescan_conversation_id(resource_span: dict) -> str | None: def _extract_conversation_id(attrs_list: list[dict]) -> str | None: """Extract gen_ai.conversation.id from OTLP span attributes.""" - return _extract_string_attribute(attrs_list, OTEL_GENAI_CONVERSATION_ID) + for attr in attrs_list: + if attr.get("key") == OTEL_GENAI_CONVERSATION_ID: + return attr.get("value", {}).get("stringValue") + return None def _convert_otlp_log_record(log_record: dict) -> dict | None: diff --git a/src/agentevals/otlp_anyvalue.py b/src/agentevals/otlp_anyvalue.py index 9f498d3..6eb8bef 100644 --- a/src/agentevals/otlp_anyvalue.py +++ b/src/agentevals/otlp_anyvalue.py @@ -6,15 +6,20 @@ ``MessageToDict``) and OTLP/JSON payloads deliver that same dict shape, so every consumer needs identical decoding rules. -This module is deliberately dependency-free: importing only the standard -library lets ``extraction``, ``loader.otlp`` and ``api.otlp_processing`` all -use it without creating an import cycle. +This module depends only on the standard library and ``trace_attrs`` (a leaf +constants module), so ``extraction``, ``loader.otlp`` and ``api.otlp_processing`` +can all use it without creating an import cycle. """ from __future__ import annotations +import logging from typing import Any +from .trace_attrs import SPEC_STRING_ATTRS + +logger = logging.getLogger(__name__) + ANY_VALUE_FIELDS = ( "stringValue", "intValue", @@ -71,10 +76,31 @@ def decode_attributes(attrs_list: list[dict]) -> dict[str, Any]: Entries whose value carries no ``AnyValue`` field are skipped, matching the behaviour every call site had before they shared this decoder. + + Attributes listed in :data:`SPEC_STRING_ATTRS` are narrowed back to + str-or-absent. Decoding the full union means an attribute the spec types as + a string can now arrive as a list or dict, and several consumers use these + values as dict keys, where an unhashable value raises ``TypeError``. + Narrowing here rather than at each call site keeps the guarantee tied to + the spec rather than to whichever consumers we have already found. + + A container in a string-typed slot is dropped rather than serialised: JSON + dumping it would put the literal blob back into user-visible output, which + is the symptom this decoder exists to remove. """ result: dict[str, Any] = {} for attr in attrs_list: value_obj = attr.get("value", {}) - if is_any_value(value_obj): - result[attr.get("key", "")] = decode_any_value(value_obj) + if not is_any_value(value_obj): + continue + key = attr.get("key", "") + value = decode_any_value(value_obj) + if key in SPEC_STRING_ATTRS and not isinstance(value, str): + logger.warning( + "Dropping non-string value for string-typed attribute %s (got %s)", + key, + type(value).__name__, + ) + continue + result[key] = value return result diff --git a/src/agentevals/trace_attrs.py b/src/agentevals/trace_attrs.py index e4cd16a..c51b550 100644 --- a/src/agentevals/trace_attrs.py +++ b/src/agentevals/trace_attrs.py @@ -84,3 +84,47 @@ # agentevals custom attributes (repository-specific, outside OTel semconv) AGENTEVALS_SESSION_ID = "agentevals.session_id" +AGENTEVALS_EVAL_SET_ID = "agentevals.eval_set_id" +AGENTEVALS_SESSION_NAME = "agentevals.session_name" + +# Attributes the GenAI semconv (and our own agentevals.* namespace) define as +# plain strings. The shared AnyValue decoder narrows these to str-or-absent so +# a container value can never reach a dict-key position such as +# ``StreamingTraceManager._active_session_for_name``. +# +# Hand-maintained on purpose: opentelemetry-semantic-conventions exposes only +# constants and prose docstrings, so the value type of an attribute is not +# machine-readable and this set cannot be derived from the package. +# +# Attributes the spec types as arrays or structured values are deliberately +# absent - gen_ai.response.finish_reasons, gen_ai.input/output.messages, +# gen_ai.tool.definitions and gen_ai.system_instructions must keep their +# decoded containers, which is the whole point of #173. Numeric attributes are +# likewise absent; the decoder already returns them as int/float. +SPEC_STRING_ATTRS: frozenset[str] = frozenset( + { + OTEL_SERVICE_NAME, + OTEL_SCOPE, + OTEL_SCOPE_VERSION, + OTEL_SCHEMA_URL, + OTEL_ERROR_TYPE, + OTEL_GENAI_OP, + OTEL_GENAI_AGENT_NAME, + OTEL_GENAI_AGENT_ID, + OTEL_GENAI_AGENT_DESCRIPTION, + OTEL_GENAI_REQUEST_MODEL, + OTEL_GENAI_RESPONSE_MODEL, + OTEL_GENAI_RESPONSE_ID, + OTEL_GENAI_PROVIDER_NAME, + OTEL_GENAI_SYSTEM, + OTEL_GENAI_CONVERSATION_ID, + OTEL_GENAI_TOOL_NAME, + OTEL_GENAI_TOOL_CALL_ID, + OTEL_GENAI_TOOL_TYPE, + OTEL_GENAI_TOOL_DESCRIPTION, + OTEL_GENAI_OUTPUT_TYPE, + AGENTEVALS_SESSION_ID, + AGENTEVALS_EVAL_SET_ID, + AGENTEVALS_SESSION_NAME, + } +) diff --git a/tests/test_extraction.py b/tests/test_extraction.py index ec3591f..6d8d4b2 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -447,6 +447,58 @@ def test_array_of_kvlist(self): ) assert result == {"gen_ai.tool.calls": [{"name": "get_weather"}]} + def test_finish_reasons_survive_to_extracted_model_info(self): + """The symptom #173 names: gen_ai.response.finish_reasons reaching the + consumer as ["stop"] rather than a literal blob or nothing. + + Asserting through extract_extended_model_info_from_attrs rather than at + the decoder keeps the whole path covered - decoding it correctly is not + the same as it arriving correctly. + """ + attrs = flatten_otlp_attributes( + [ + { + "key": "gen_ai.response.finish_reasons", + "value": {"arrayValue": {"values": [{"stringValue": "stop"}]}}, + }, + {"key": "gen_ai.response.model", "value": {"stringValue": "claude-opus-5"}}, + ] + ) + info = extract_extended_model_info_from_attrs(attrs) + assert info["finish_reasons"] == ["stop"] + assert info["response_model"] == "claude-opus-5" + + def test_multiple_finish_reasons_survive(self): + attrs = flatten_otlp_attributes( + [ + { + "key": "gen_ai.response.finish_reasons", + "value": {"arrayValue": {"values": [{"stringValue": "stop"}, {"stringValue": "length"}]}}, + } + ] + ) + assert extract_extended_model_info_from_attrs(attrs)["finish_reasons"] == [ + "stop", + "length", + ] + + def test_string_typed_attribute_drops_container_value(self): + """gen_ai.response.model is typed as a string by the spec; a container + in that slot is dropped rather than carried into consumers.""" + attrs = flatten_otlp_attributes( + [ + { + "key": "gen_ai.response.model", + "value": {"arrayValue": {"values": [{"stringValue": "claude-opus-5"}]}}, + }, + {"key": "gen_ai.request.model", "value": {"stringValue": "claude-sonnet-5"}}, + ] + ) + assert "gen_ai.response.model" not in attrs + info = extract_extended_model_info_from_attrs(attrs) + assert info["response_model"] is None + assert info["request_model"] == "claude-sonnet-5" + def test_bytes_value(self): """MessageToDict base64-encodes bytes fields, so the decoder sees a str.""" result = flatten_otlp_attributes([{"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}]) diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index 031c51e..ec0572d 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -7,6 +7,7 @@ import pytest +from agentevals.extraction import extract_user_text_from_attrs from agentevals.loader.otlp import OtlpJsonLoader @@ -376,6 +377,34 @@ def test_event_promotion_decodes_array_value(self): ) assert span.tags["gen_ai.input.messages"] == [{"role": "user", "content": "Hello"}] + def test_promoted_array_messages_reach_the_consumer(self): + """Past span.tags: a complex-array gen_ai.input.messages promoted out of + a span event must still yield the user text downstream, which is what a + consumer actually reads.""" + span = self._load_span_with_event_attribute( + { + "key": "gen_ai.input.messages", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + {"key": "role", "value": {"stringValue": "user"}}, + { + "key": "content", + "value": {"stringValue": "What is the weather?"}, + }, + ] + } + } + ] + } + }, + } + ) + assert extract_user_text_from_attrs(span.tags) == "What is the weather?" + def test_event_promotion_keeps_string_value(self): """The pre-existing stringValue path must keep working unchanged.""" messages_json = '[{"role": "user", "content": "Hello"}]' diff --git a/tests/test_otlp_receiver.py b/tests/test_otlp_receiver.py index 6ec0f70..a9c7a22 100644 --- a/tests/test_otlp_receiver.py +++ b/tests/test_otlp_receiver.py @@ -261,17 +261,31 @@ def test_non_string_eval_set_id_is_ignored(self): meta = _extract_agentevals_metadata(attrs) assert meta["eval_set_id"] is None - def test_non_string_values_still_reach_resource_attrs(self): - """Narrowing applies to the two key-like fields only; the decoded value - is still available in resource_attrs for anything that wants it.""" + def test_non_string_values_are_dropped_from_resource_attrs(self): + """Narrowing happens in the decoder, so a container in a string-typed + slot never lands in resource_attrs either.""" attrs = [ { "key": "agentevals.session_name", "value": {"arrayValue": {"values": [{"stringValue": "run-42"}]}}, + }, + _make_otlp_attr("service.name", "test-agent"), + ] + meta = _extract_agentevals_metadata(attrs) + assert "agentevals.session_name" not in meta["resource_attrs"] + assert meta["resource_attrs"]["service.name"] == "test-agent" + + def test_spec_array_attributes_keep_their_container(self): + """The narrowing must not touch attributes the spec types as arrays - + that is what #173 fixes.""" + attrs = [ + { + "key": "gen_ai.response.finish_reasons", + "value": {"arrayValue": {"values": [{"stringValue": "stop"}]}}, } ] meta = _extract_agentevals_metadata(attrs) - assert meta["resource_attrs"]["agentevals.session_name"] == ["run-42"] + assert meta["resource_attrs"]["gen_ai.response.finish_reasons"] == ["stop"] # --------------------------------------------------------------------------- From 471301c4c017f8d1dc7750379aee8ad31d9662cb Mon Sep 17 00:00:00 2001 From: Leroyyyyyyyyy <150530443+Leroyyyyyyyyy@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:14:19 +0800 Subject: [PATCH 5/5] fix(otel): decode containers on an opt-in allowlist instead of a denylist SPEC_STRING_ATTRS listed the attributes to narrow, so a missing entry let an unhashable value reach a dict key on an unauthenticated receiver port. It missed the whole gcp.vertex.agent.* namespace. Replace it with SPEC_CONTAINER_ATTRS: the seven keys the GenAI semconv types as arrays or structured values. Those decode to native containers; every other key decodes to a scalar or is dropped, which is what extraction.py did before the decoder was shared. A missing entry now drops one value rather than crashing ingestion, and attributes neither of us has thought about are safe by construction rather than by enumeration. decode_attribute() carries the rule so the OTLP-array path and the span-event promotion path share one implementation. bytesValue now survives as the base64 string it already was, which #173 asks for and is hashable. The default for unlisted container values is tracked in #208; the nested-dict loader path that bypasses this decoder entirely is #207. --- src/agentevals/api/otlp_processing.py | 4 +- src/agentevals/loader/otlp.py | 6 ++- src/agentevals/otlp_anyvalue.py | 56 ++++++++++++++++---------- src/agentevals/trace_attrs.py | 58 ++++++++++----------------- tests/test_extraction.py | 42 ++++++++++++++----- tests/test_otlp_loader.py | 4 +- tests/test_otlp_receiver.py | 8 ++-- 7 files changed, 99 insertions(+), 79 deletions(-) diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 297728a..5aefedd 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -246,8 +246,8 @@ def _extract_agentevals_metadata(resource_attrs: list[dict]) -> dict: ``eval_set_id`` and ``session_name`` are used as dict keys downstream (``_active_session_for_name``) and typed ``str | None`` on the session - models. Both keys are in ``SPEC_STRING_ATTRS``, so the shared decoder has - already narrowed them to str-or-absent by the time they get here. + models. Neither is in ``SPEC_CONTAINER_ATTRS``, so the shared decoder never + hands them a list or dict in the first place. """ flat = flatten_otlp_attributes(resource_attrs) return { diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index 24a4378..de680fd 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -5,7 +5,7 @@ import json import logging -from ..otlp_anyvalue import decode_any_value, decode_attributes, is_any_value +from ..otlp_anyvalue import decode_attribute, decode_attributes, is_any_value from ..trace_attrs import ( OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, @@ -177,7 +177,9 @@ def _promote_genai_event_attributes(self, span_data: dict, attributes: dict) -> if key in self._GENAI_EVENT_KEYS and key not in attributes: value_obj = attr.get("value", {}) if is_any_value(value_obj): - attributes[key] = decode_any_value(value_obj) + keep, value = decode_attribute(key, value_obj) + if keep: + attributes[key] = value def _extract_attributes(self, attrs) -> dict: """Convert attributes to a flat ``{key: value}`` dict. diff --git a/src/agentevals/otlp_anyvalue.py b/src/agentevals/otlp_anyvalue.py index 6eb8bef..923a6a7 100644 --- a/src/agentevals/otlp_anyvalue.py +++ b/src/agentevals/otlp_anyvalue.py @@ -16,7 +16,7 @@ import logging from typing import Any -from .trace_attrs import SPEC_STRING_ATTRS +from .trace_attrs import SPEC_CONTAINER_ATTRS logger = logging.getLogger(__name__) @@ -71,22 +71,40 @@ def is_any_value(value_obj: dict) -> bool: return False +def decode_attribute(key: str, value_obj: dict) -> tuple[bool, Any]: + """Decode one attribute, applying the container allowlist. + + Returns ``(keep, value)``. Scalars are always kept. A list or dict is kept + only when *key* is in :data:`SPEC_CONTAINER_ATTRS`; otherwise it is dropped, + which is what ``extraction.py`` did with containers before this decoder was + shared. + + Dropping rather than serialising is deliberate: JSON-dumping the value would + put a blob back into user-visible output, which is the symptom #173 is + about. What the default *should* be is tracked in #208. + """ + value = decode_any_value(value_obj) + if isinstance(value, (list, dict)) and key not in SPEC_CONTAINER_ATTRS: + logger.warning( + "Dropping container value for %s (got %s); only spec container attributes are kept", + key, + type(value).__name__, + ) + return False, None + return True, value + + def decode_attributes(attrs_list: list[dict]) -> dict[str, Any]: """Decode an OTLP attributes array to a flat ``{key: value}`` dict. - Entries whose value carries no ``AnyValue`` field are skipped, matching - the behaviour every call site had before they shared this decoder. - - Attributes listed in :data:`SPEC_STRING_ATTRS` are narrowed back to - str-or-absent. Decoding the full union means an attribute the spec types as - a string can now arrive as a list or dict, and several consumers use these - values as dict keys, where an unhashable value raises ``TypeError``. - Narrowing here rather than at each call site keeps the guarantee tied to - the spec rather than to whichever consumers we have already found. + Entries whose value carries no ``AnyValue`` field are skipped, matching the + behaviour every call site had before they shared this decoder. - A container in a string-typed slot is dropped rather than serialised: JSON - dumping it would put the literal blob back into user-visible output, which - is the symptom this decoder exists to remove. + Container values survive only for the keys in + :data:`~agentevals.trace_attrs.SPEC_CONTAINER_ATTRS`. Keeping the allowlist + here rather than narrowing per consumer means an attribute nobody has + thought about cannot become an unhashable dict key downstream - there is + nothing to remember, because it was never widened in the first place. """ result: dict[str, Any] = {} for attr in attrs_list: @@ -94,13 +112,7 @@ def decode_attributes(attrs_list: list[dict]) -> dict[str, Any]: if not is_any_value(value_obj): continue key = attr.get("key", "") - value = decode_any_value(value_obj) - if key in SPEC_STRING_ATTRS and not isinstance(value, str): - logger.warning( - "Dropping non-string value for string-typed attribute %s (got %s)", - key, - type(value).__name__, - ) - continue - result[key] = value + keep, value = decode_attribute(key, value_obj) + if keep: + result[key] = value return result diff --git a/src/agentevals/trace_attrs.py b/src/agentevals/trace_attrs.py index c51b550..bb430e8 100644 --- a/src/agentevals/trace_attrs.py +++ b/src/agentevals/trace_attrs.py @@ -87,44 +87,30 @@ AGENTEVALS_EVAL_SET_ID = "agentevals.eval_set_id" AGENTEVALS_SESSION_NAME = "agentevals.session_name" -# Attributes the GenAI semconv (and our own agentevals.* namespace) define as -# plain strings. The shared AnyValue decoder narrows these to str-or-absent so -# a container value can never reach a dict-key position such as -# ``StreamingTraceManager._active_session_for_name``. +# Attributes the GenAI semconv types as arrays or structured values. These are +# the only keys the shared AnyValue decoder returns as a native list or dict; +# every other key is decoded to a scalar or dropped, which is what +# ``extraction.py`` did before the decoder was shared. # -# Hand-maintained on purpose: opentelemetry-semantic-conventions exposes only -# constants and prose docstrings, so the value type of an attribute is not -# machine-readable and this set cannot be derived from the package. +# The set is an allowlist on purpose. A missing entry drops one value, matching +# the pre-existing extraction behaviour; a denylist would instead let an +# unhashable value reach a dict key or set member on an unauthenticated +# receiver port, which crashes ingestion. See the discussion on #187. # -# Attributes the spec types as arrays or structured values are deliberately -# absent - gen_ai.response.finish_reasons, gen_ai.input/output.messages, -# gen_ai.tool.definitions and gen_ai.system_instructions must keep their -# decoded containers, which is the whole point of #173. Numeric attributes are -# likewise absent; the decoder already returns them as int/float. -SPEC_STRING_ATTRS: frozenset[str] = frozenset( +# Hand-maintained: opentelemetry-semantic-conventions exposes constants and +# prose docstrings only, so attribute value types are not machine-readable and +# this set cannot be derived from the package. +# +# What should happen to container values on keys outside this set is not +# settled; today they are dropped. Tracked in #208. +SPEC_CONTAINER_ATTRS: frozenset[str] = frozenset( { - OTEL_SERVICE_NAME, - OTEL_SCOPE, - OTEL_SCOPE_VERSION, - OTEL_SCHEMA_URL, - OTEL_ERROR_TYPE, - OTEL_GENAI_OP, - OTEL_GENAI_AGENT_NAME, - OTEL_GENAI_AGENT_ID, - OTEL_GENAI_AGENT_DESCRIPTION, - OTEL_GENAI_REQUEST_MODEL, - OTEL_GENAI_RESPONSE_MODEL, - OTEL_GENAI_RESPONSE_ID, - OTEL_GENAI_PROVIDER_NAME, - OTEL_GENAI_SYSTEM, - OTEL_GENAI_CONVERSATION_ID, - OTEL_GENAI_TOOL_NAME, - OTEL_GENAI_TOOL_CALL_ID, - OTEL_GENAI_TOOL_TYPE, - OTEL_GENAI_TOOL_DESCRIPTION, - OTEL_GENAI_OUTPUT_TYPE, - AGENTEVALS_SESSION_ID, - AGENTEVALS_EVAL_SET_ID, - AGENTEVALS_SESSION_NAME, + OTEL_GENAI_RESPONSE_FINISH_REASONS, + OTEL_GENAI_INPUT_MESSAGES, + OTEL_GENAI_OUTPUT_MESSAGES, + OTEL_GENAI_TOOL_DEFINITIONS, + OTEL_GENAI_SYSTEM_INSTRUCTIONS, + OTEL_GENAI_TOOL_CALL_ARGUMENTS, + OTEL_GENAI_TOOL_CALL_RESULT, } ) diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 6d8d4b2..8733262 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -6,6 +6,7 @@ import pytest +from agentevals import trace_attrs from agentevals.extraction import ( AdkExtractor, GenAIExtractor, @@ -415,37 +416,37 @@ def test_kvlist_value(self): result = flatten_otlp_attributes( [ { - "key": "gen_ai.request.params", + "key": "gen_ai.tool.call.arguments", "value": { "kvlistValue": { "values": [ - {"key": "temperature", "value": {"doubleValue": 0.7}}, - {"key": "stream", "value": {"boolValue": False}}, + {"key": "city", "value": {"stringValue": "Berlin"}}, + {"key": "metric", "value": {"boolValue": False}}, ] } }, }, ] ) - assert result == {"gen_ai.request.params": {"temperature": 0.7, "stream": False}} + assert result == {"gen_ai.tool.call.arguments": {"city": "Berlin", "metric": False}} def test_array_of_kvlist(self): - """Tool calls arrive as an arrayValue of kvlistValue.""" + """Messages arrive as an arrayValue of kvlistValue.""" result = flatten_otlp_attributes( [ { - "key": "gen_ai.tool.calls", + "key": "gen_ai.input.messages", "value": { "arrayValue": { "values": [ - {"kvlistValue": {"values": [{"key": "name", "value": {"stringValue": "get_weather"}}]}}, + {"kvlistValue": {"values": [{"key": "role", "value": {"stringValue": "user"}}]}}, ] } }, }, ] ) - assert result == {"gen_ai.tool.calls": [{"name": "get_weather"}]} + assert result == {"gen_ai.input.messages": [{"role": "user"}]} def test_finish_reasons_survive_to_extracted_model_info(self): """The symptom #173 names: gen_ai.response.finish_reasons reaching the @@ -482,9 +483,9 @@ def test_multiple_finish_reasons_survive(self): "length", ] - def test_string_typed_attribute_drops_container_value(self): - """gen_ai.response.model is typed as a string by the spec; a container - in that slot is dropped rather than carried into consumers.""" + def test_unlisted_key_drops_container_value(self): + """Containers survive only for SPEC_CONTAINER_ATTRS. Everything else is + dropped, which is what extraction did before the decoder was shared.""" attrs = flatten_otlp_attributes( [ { @@ -499,6 +500,25 @@ def test_string_typed_attribute_drops_container_value(self): assert info["response_model"] is None assert info["request_model"] == "claude-sonnet-5" + def test_no_unlisted_key_can_yield_an_unhashable_value(self): + """The property the allowlist exists for: nothing outside + SPEC_CONTAINER_ATTRS can reach a consumer as a dict key or set member + and raise TypeError. Covers every attribute constant we declare, so a + new one cannot quietly reopen the hazard.""" + container = {"arrayValue": {"values": [{"stringValue": "x"}]}} + for name in dir(trace_attrs): + if not name.isupper(): + continue + key = getattr(trace_attrs, name) + if not isinstance(key, str): + continue + value = flatten_otlp_attributes([{"key": key, "value": container}]).get(key) + if key in trace_attrs.SPEC_CONTAINER_ATTRS: + assert value == ["x"], f"{key} should keep its container" + else: + assert value is None, f"{key} leaked a container" + hash(value) + def test_bytes_value(self): """MessageToDict base64-encodes bytes fields, so the decoder sees a str.""" result = flatten_otlp_attributes([{"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}]) diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index ec0572d..2b2f27e 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -301,7 +301,7 @@ def test_array_value(self): def test_kvlist_value(self): span = self._load_span_with( { - "key": "gen_ai.request.params", + "key": "gen_ai.tool.call.arguments", "value": { "kvlistValue": { "values": [ @@ -312,7 +312,7 @@ def test_kvlist_value(self): }, } ) - assert span.tags["gen_ai.request.params"] == {"temperature": 0.7, "stream": False} + assert span.tags["gen_ai.tool.call.arguments"] == {"temperature": 0.7, "stream": False} def test_bytes_value(self): span = self._load_span_with({"key": "payload", "value": {"bytesValue": "AP9oaQ=="}}) diff --git a/tests/test_otlp_receiver.py b/tests/test_otlp_receiver.py index a9c7a22..ae8cc56 100644 --- a/tests/test_otlp_receiver.py +++ b/tests/test_otlp_receiver.py @@ -262,8 +262,8 @@ def test_non_string_eval_set_id_is_ignored(self): assert meta["eval_set_id"] is None def test_non_string_values_are_dropped_from_resource_attrs(self): - """Narrowing happens in the decoder, so a container in a string-typed - slot never lands in resource_attrs either.""" + """agentevals.session_name is not in SPEC_CONTAINER_ATTRS, so a container + never lands in resource_attrs and can never reach a dict key.""" attrs = [ { "key": "agentevals.session_name", @@ -276,8 +276,8 @@ def test_non_string_values_are_dropped_from_resource_attrs(self): assert meta["resource_attrs"]["service.name"] == "test-agent" def test_spec_array_attributes_keep_their_container(self): - """The narrowing must not touch attributes the spec types as arrays - - that is what #173 fixes.""" + """Keys in SPEC_CONTAINER_ATTRS keep their decoded container - that is + what #173 fixes.""" attrs = [ { "key": "gen_ai.response.finish_reasons",