From a01ba3d4be13a018ca1d2892cba54e9aceeb4344 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 16 Oct 2026 15:59:48 +0200 Subject: [PATCH 1/4] fix(server-utils): Keep OpenAI decision confidence on Vercel AI decide spans Co-Authored-By: Claude Opus 5.5 --- .../vercel-ai/vercel-ai-dc-subscriber.ts | 10 ++++--- .../integrations/vercel-ai/decide.test.ts | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index fe102c833e88..f16d1f5eee02 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -701,13 +701,15 @@ function getOutputMessages( } /** - * The AI SDK TypeSafe provider moves each answer's `confidence` out of the answers into - * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so decide answers keep it. + * The AI SDK TypeSafe and OpenAI providers move each answer's `confidence` out of the answers into + * `providerMetadata..confidence` (keyed by question id). Put it back so decide answers keep it. */ function withProviderConfidence(result: Record): unknown { const { answers, providerMetadata } = result; - const typesafe = isObjectLike(providerMetadata) ? providerMetadata.typesafe : undefined; - const confidence = isObjectLike(typesafe) && isObjectLike(typesafe.confidence) ? typesafe.confidence : undefined; + const metadata = isObjectLike(providerMetadata) + ? [providerMetadata.typesafe, providerMetadata.openai].find(m => isObjectLike(m) && isObjectLike(m.confidence)) + : undefined; + const confidence = isObjectLike(metadata) && isObjectLike(metadata.confidence) ? metadata.confidence : undefined; if (!confidence || !isObjectLike(answers)) { return answers; } diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index 3037a264f092..eb7933bf49b2 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -55,4 +55,31 @@ describe('Vercel AI decide spans', () => { { type: 'evaluation', answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } }, ]); }); + + it.each(['typesafe', 'openai'])('restores answer confidence from `providerMetadata.%s`', provider => { + const message = { + type: 'experimental_decide', + event: { + provider, + modelId: 'gpt-6-luna', + state: 'I was charged twice', + questions: { department: { type: 'choice', criteria: { billing: 'Charges', support: 'Other' } } }, + recordInputs: true, + recordOutputs: true, + }, + result: { + answers: { department: { type: 'choice', value: 'billing' } }, + providerMetadata: { [provider]: { confidence: { department: 0.91 } } }, + }, + } as unknown as Message; + + const span = createSpanFromMessage(message, {} as Parameters[1])!; + enrichSpanOnEnd(span, message, {} as Parameters[2]); + span.end(); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([ + { type: 'evaluation', answers: { department: { type: 'choice', value: 'billing', confidence: 0.91 } } }, + ]); + }); }); From 8ac6f648ce6b823324f9f77e84a7a1b98ce4c5d5 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 19 Oct 2026 11:26:13 +0200 Subject: [PATCH 2/4] Use choice field in decide confidence test --- .../server-utils/test/integrations/vercel-ai/decide.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index eb7933bf49b2..ca259a765a36 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -68,7 +68,7 @@ describe('Vercel AI decide spans', () => { recordOutputs: true, }, result: { - answers: { department: { type: 'choice', value: 'billing' } }, + answers: { department: { type: 'choice', choice: 'billing' } }, providerMetadata: { [provider]: { confidence: { department: 0.91 } } }, }, } as unknown as Message; @@ -79,7 +79,7 @@ describe('Vercel AI decide spans', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([ - { type: 'evaluation', answers: { department: { type: 'choice', value: 'billing', confidence: 0.91 } } }, + { type: 'evaluation', answers: { department: { type: 'choice', choice: 'billing', confidence: 0.91 } } }, ]); }); }); From 8aca618a942e2b60a7da01678aa7a0c219a96015 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 19 Oct 2026 11:51:28 +0200 Subject: [PATCH 3/4] Use conventions constant in decide test assertions --- .../server-utils/test/integrations/vercel-ai/decide.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index ca259a765a36..4bfec2e6497f 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; @@ -51,7 +52,7 @@ describe('Vercel AI decide spans', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('decide typesafe-ai/jev'); expect(json.op).toBe(GEN_AI_DECIDE); - expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([ + expect(JSON.parse(json.data?.[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } }, ]); }); @@ -78,7 +79,7 @@ describe('Vercel AI decide spans', () => { span.end(); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([ + expect(JSON.parse(json.data?.[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { department: { type: 'choice', choice: 'billing', confidence: 0.91 } } }, ]); }); From 274aa372ca0307ce5bc90308088a0c1f5ea75147 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 21 Oct 2026 11:18:47 +0200 Subject: [PATCH 4/4] Use satisfies for decide test messages --- .../server-utils/test/integrations/vercel-ai/decide.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index 4bfec2e6497f..ae668d64e338 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -42,7 +42,7 @@ describe('Vercel AI decide spans', () => { recordOutputs: true, }, result: { answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } }, - } as unknown as Message; + } satisfies Message; const span = createSpanFromMessage(message, {} as Parameters[1])!; expect(span).toBeDefined(); @@ -72,7 +72,7 @@ describe('Vercel AI decide spans', () => { answers: { department: { type: 'choice', choice: 'billing' } }, providerMetadata: { [provider]: { confidence: { department: 0.91 } } }, }, - } as unknown as Message; + } satisfies Message; const span = createSpanFromMessage(message, {} as Parameters[1])!; enrichSpanOnEnd(span, message, {} as Parameters[2]);