test(e2e): Cover Vercel AI decisions in node-openai-decisions-send-to-sentry

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-openai-decisions-send-to-sentry",
-  "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project",
+  "description": "The OpenAI Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions requests against OpenAI through the openai client and the Vercel AI SDK, and the gen_ai.decide spans read back from a real Sentry project",
   "version": "1.0.0",
   "private": true,
   "type": "module",
@@ -10,13 +10,16 @@
     "test": "playwright test",
     "clean": "npx rimraf node_modules dist pnpm-lock.yaml",
     "test:build": "pnpm install && pnpm build",
-    "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build",
+    "test:build-latest": "pnpm install && pnpm add openai@latest ai@latest @ai-sdk/openai@latest && pnpm build",
     "test:assert": "pnpm test"
   },
   "dependencies": {
+    "@ai-sdk/openai": "4.0.95",
     "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
+    "ai": "7.0.142",
     "express": "^4.21.2",
-    "openai": "7.32.0"
+    "openai": "7.32.0",
+    "zod": "^4.1.8"
   },
   "devDependencies": {
     "@playwright/test": "1.64.1",

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts

@@ -1,5 +1,7 @@
 // `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here.
+import { createOpenAI } from '@ai-sdk/openai';
 import * as Sentry from '@sentry/node';
+import { experimental_decide } from 'ai';
 import express from 'express';
 import OpenAI from 'openai';
 
@@ -11,6 +13,7 @@ if (!apiKey) {
 }
 
 const client = new OpenAI({ apiKey });
+const aiSdkOpenAI = createOpenAI({ apiKey });
 
 const MODEL = 'gpt-6-luna';
 
@@ -74,6 +77,29 @@ app.get('/decide-image', async (_req, res, next) => {
   }
 });
 
+// The same decision through the Vercel AI SDK, which calls the Decisions API itself rather than through
+// the `openai` client, so the Vercel AI integration records it.
+app.get('/vercel-ai-decide', async (_req, res, next) => {
+  try {
+    const { answers } = await experimental_decide({
+      model: aiSdkOpenAI.decisionModel(MODEL),
+      state: 'I was charged twice this month and want a refund.',
+      questions: {
+        wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
+        department: {
+          type: 'choice',
+          instructions: 'Which team should handle this?',
+          criteria: { billing: 'Charges and refunds', other: 'Anything else' },
+        },
+        urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
+      },
+    });
+    res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers });
+  } catch (error) {
+    next(error);
+  }
+});
+
 app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
   res.status(500).send({ message: error.message });
 });

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts

@@ -157,3 +157,68 @@ test('records an image decision with its image data', async ({ baseURL }) => {
     { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } },
   ]);
 });
+
+test('records a Vercel AI decision with the confidence of its answers', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/vercel-ai-decide`);
+  const body = await response.json();
+  expect(response.status, JSON.stringify(body)).toBe(200);
+  const { traceId }: { traceId: string } = body;
+
+  console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
+
+  let spanId: string | undefined;
+  await expect
+    .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS)
+    .toBeDefined();
+
+  let attributes: Record<string, unknown> | undefined;
+  await expect
+    .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS)
+    .toMatchObject({
+      'span.name': `decide ${MODEL}`,
+      [SEARCH_SPAN__OP]: GEN_AI_DECIDE,
+      [SEARCH_SPAN__STATUS]: 'ok',
+      [SEARCH_ORIGIN]: 'auto.vercelai.channel',
+      [SEARCH_GEN_AI__OPERATION__NAME]: 'decide',
+      [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai.decision',
+      [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL,
+    });
+
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([
+    {
+      type: 'evaluation',
+      state: 'I was charged twice this month and want a refund.',
+      questions: {
+        wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
+        department: {
+          type: 'choice',
+          instructions: 'Which team should handle this?',
+          criteria: { billing: 'Charges and refunds', other: 'Anything else' },
+        },
+        urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
+      },
+    },
+  ]);
+
+  // The OpenAI provider moves `confidence` into its provider metadata; the span must keep it on the answers.
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([
+    {
+      type: 'evaluation',
+      answers: {
+        wantsRefund: { type: 'boolean', probability: expect.any(Number) },
+        department: {
+          type: 'choice',
+          choice: expect.stringMatching(/^(billing|other)$/),
+          probabilities: { billing: expect.any(Number), other: expect.any(Number) },
+          confidence: expect.any(Number),
+        },
+        urgency: {
+          type: 'score',
+          score: expect.any(Number),
+          probabilities: { 0: expect.any(Number), 1: expect.any(Number) },
+          confidence: expect.any(Number),
+        },
+      },
+    },
+  ]);
+});