diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore new file mode 100644 index 000000000000..2685f6ec088a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore @@ -0,0 +1,7 @@ +node_modules +dist +.flue +*.tsbuildinfo +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts new file mode 100644 index 000000000000..a31c8523a9a7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from '@flue/runtime/config'; + +export default defineConfig({ + target: 'node', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json new file mode 100644 index 000000000000..409652d5c19e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json @@ -0,0 +1,49 @@ +{ + "name": "node-flue-send-to-sentry", + "description": "The Flue integration as a user runs it: a built Flue agent server, real tool-calling turns through OpenRouter, and the gen_ai spans read back from a real Sentry project", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "vite build", + "start": "NODE_OPTIONS='--import=@sentry/node/import' PORT=3030 node dist/server.mjs", + "test": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add @flue/runtime@latest @flue/vite@latest @flue/cli@latest && pnpm build", + "test:assert": "pnpm test" + }, + "dependencies": { + "@flue/runtime": "2.0.5", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "dataloader": "^2.2.3", + "hono": "^4.7.0", + "valibot": "^1.5.0" + }, + "devDependencies": { + "@flue/cli": "2.0.5", + "@flue/vite": "2.0.5", + "@playwright/test": "1.64.1", + "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz", + "@sentry/conventions": "0.27.0", + "@types/node": "24.x", + "sentry": "~0.44.1", + "typescript": "~5.9.0", + "vite": "^8.0.14" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-flue-send-to-sentry (latest)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts new file mode 100644 index 000000000000..161e4b1c5d72 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from '@playwright/test'; + +const port = 3030; + +export default defineConfig({ + testDir: './tests', + // A model turn, then Sentry polls of up to 180s each. Spans become queryable within the first one. + timeout: 400_000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + // Every test spends most of its time polling Sentry, so run them all at once. + workers: '100%', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + use: { + baseURL: `http://localhost:${port}`, + }, + webServer: { + command: 'pnpm start', + port, + stdout: 'pipe', + stderr: 'pipe', + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml new file mode 100644 index 000000000000..0e523a16e6f0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml @@ -0,0 +1,6 @@ +allowBuilds: + '@google/genai': false + core-js-pure: false + esbuild: true + protobufjs: false +minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts new file mode 100644 index 000000000000..30f0b4c8cd49 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts @@ -0,0 +1,11 @@ +import { instrument } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; + +// No tunnel: the spans go to the Sentry project behind `E2E_TEST_DSN`, where the tests read them back. +Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: process.env.E2E_TEST_DSN, + tracesSampleRate: 1.0, +}); + +instrument(Sentry.createFlueInstrumentation()); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts new file mode 100644 index 000000000000..9d218454a86e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts @@ -0,0 +1,50 @@ +'use agent'; +import { useModel, useTool } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; +import * as v from 'valibot'; +import DataLoader from 'dataloader'; + +export function Hello() { + useModel('openrouter/anthropic/claude-haiku-4.5'); + + useTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + input: v.object({ city: v.string() }), + // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, + // so this should nest directly under it rather than landing beside it. + run: ({ data }) => + Sentry.startSpan( + { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': data.city } }, + () => { + return `It is 21 degrees and sunny in ${data.city}.`; + }, + ), + }); + + // Called from inside a tool on purpose: the dataloader span then lands under `execute_tool` in + // the agent's trace, which is what "captured alongside the AI spans" has to mean. + useTool({ + name: 'count_items', + description: 'Count items by loading them. Call this when the user asks to count items.', + input: v.object({}), + // Constructed per execution, like node-eve does: a module-level loader caches its keys, so a + // second call would skip the batch function and emit no span. + run: async () => { + const loader = new DataLoader(async keys => keys.map(key => key * 2)); + const doubled = await Promise.all([loader.load(1), loader.load(2), loader.load(3)]); + return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`; + }, + }); + + useTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + input: v.object({}), + run: () => { + throw new Error('Intentional flue tool failure'); + }, + }); + + return 'You are a helpful assistant. Use get_weather when asked about weather, count_items when asked to count items, and fail_now when asked to fail.'; +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts new file mode 100644 index 000000000000..b54a3357e4ca --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts @@ -0,0 +1,10 @@ +import '../sentry-init.ts'; +import { createAgentRouter } from '@flue/runtime/routing'; +import { Hono } from 'hono'; +import { Hello } from './agents/hello.ts'; + +const app = new Hono(); + +app.route('/agents/hello', createAgentRouter(Hello)); + +export default app; diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts new file mode 100644 index 000000000000..ba6a62fc8f7d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from '@playwright/test'; +import { SEARCH_ORIGIN } from '@sentry/conventions/attributes/search'; +import { CACHE_GET } from '@sentry/conventions/op'; +import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +// Orchestrion only hooks `dataloader` with the loader preloaded (`--import` in the start script). +test('sends orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => { + const conversationId = await runAgentTurn( + baseURL!, + newInstanceId('dataloader'), + 'Please call count_items to count the items.', + ); + + const { traceId, spans } = await waitForConversationTrace( + conversationId, + spans => + spans.some(span => span.description === 'execute_tool count_items') && spans.some(span => span.op === CACHE_GET), + ); + + const dataloaderSpan = spans.find(span => span.op === CACHE_GET); + await expect + .poll(() => fetchSpanAttributes(traceId, dataloaderSpan!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ [SEARCH_ORIGIN]: 'auto.db.dataloader' }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts new file mode 100644 index 000000000000..aa4666c1a3bf --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from '@playwright/test'; +import { SEARCH_GEN_AI__TOOL__NAME, SEARCH_SPAN__STATUS } from '@sentry/conventions/attributes/search'; +import { + EVENT_POLLING_OPTIONS, + fetchEvent, + fetchSpanAttributes, + findErrorInTrace, +} from '@sentry-internal/test-utils/cli'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +test('sends an error thrown inside a Flue tool and marks its span errored', async ({ baseURL }) => { + const conversationId = await runAgentTurn( + baseURL!, + newInstanceId('failure'), + 'Please call fail_now to trigger a failure.', + ); + + const { traceId, spans } = await waitForConversationTrace(conversationId, spans => + spans.some(span => span.description === 'execute_tool fail_now'), + ); + + let error: TraceItem | undefined; + await expect.poll(() => (error = findErrorInTrace(traceId)), EVENT_POLLING_OPTIONS).toBeDefined(); + await expect + .poll( + () => fetchEvent(error!.event_id!)?.entries.find(entry => entry.type === 'exception')?.data.values?.[0], + EVENT_POLLING_OPTIONS, + ) + .toMatchObject({ value: 'Intentional flue tool failure', mechanism: { type: 'auto.ai.flue' } }); + + const executeTool = spans.find(span => span.description === 'execute_tool fail_now'); + await expect + .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ [SEARCH_GEN_AI__TOOL__NAME]: 'fail_now', [SEARCH_SPAN__STATUS]: 'error' }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts new file mode 100644 index 000000000000..6610ad009546 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts @@ -0,0 +1,105 @@ +import { expect, test } from '@playwright/test'; +import { + SEARCH_GEN_AI__AGENT__NAME, + SEARCH_GEN_AI__CONVERSATION__ID, + SEARCH_GEN_AI__COST__TOTAL_TOKENS, + SEARCH_GEN_AI__INPUT__MESSAGES, + SEARCH_GEN_AI__OPERATION__NAME, + SEARCH_GEN_AI__PROVIDER__NAME, + SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS, + SEARCH_GEN_AI__TOOL__CALL__RESULT, + SEARCH_GEN_AI__TOOL__NAME, + SEARCH_GEN_AI__USAGE__INPUT_TOKENS, + SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, + SEARCH_ORIGIN, + SEARCH_SPAN__STATUS, +} from '@sentry/conventions/attributes/search'; +import { + GEN_AI_CHAT, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, + HTTP_CLIENT, + HTTP_SERVER, +} from '@sentry/conventions/op'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; +import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue turn', async ({ baseURL }) => { + const conversationId = await runAgentTurn(baseURL!, newInstanceId('weather'), 'What is the weather in Paris?'); + + // Spans land at different times, so wait for the relationships checked below, not just the ops. + const isProviderCall = (span: TraceItem, spans: TraceItem[]) => + span.op === HTTP_CLIENT && spans.some(chat => chat.op === GEN_AI_CHAT && chat.event_id === span.parent_span_id); + const { traceId, spans } = await waitForConversationTrace( + conversationId, + spans => + [HTTP_SERVER, GEN_AI_INVOKE_AGENT, GEN_AI_EXECUTE_TOOL].every(op => spans.some(span => span.op === op)) && + spans.some(span => isProviderCall(span, spans)), + ); + + const request = spans.find(span => span.op === HTTP_SERVER); + const agents = spans.filter(span => span.op === GEN_AI_INVOKE_AGENT); + const invokeAgent = agents[0]; + const chats = spans.filter(span => span.op === GEN_AI_CHAT); + const executeTool = spans.find(span => span.description === 'execute_tool get_weather'); + + // More than one means other conversations leaked into this trace (withastro/flue#813). + expect(agents).toHaveLength(1); + expect(invokeAgent?.parent_span_id).toBe(request?.event_id); + // A chat span anywhere else would be a provider SDK reporting the same call a second time. + expect(chats.map(chat => chat.parent_span_id)).toEqual(chats.map(() => invokeAgent?.event_id)); + expect(executeTool?.parent_span_id).toBe(invokeAgent?.event_id); + + await expect + .poll(() => fetchSpanAttributes(traceId, invokeAgent!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_GEN_AI__OPERATION__NAME]: 'invoke_agent', + [SEARCH_GEN_AI__AGENT__NAME]: 'Hello', + [SEARCH_GEN_AI__CONVERSATION__ID]: conversationId, + }); + + let chatAttributes: Record | undefined; + await expect + .poll(() => (chatAttributes = fetchSpanAttributes(traceId, chats[0]!.event_id!)), EVENT_POLLING_OPTIONS) + .toMatchObject({ + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_SPAN__STATUS]: 'ok', + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openrouter', + [SEARCH_GEN_AI__INPUT__MESSAGES]: expect.stringContaining('What is the weather in Paris?'), + }); + // The trace-items API returns integer attributes as strings. + expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBeGreaterThan(0); + // Flue computes cost itself; no provider SDK reports it. + expect(chatAttributes![SEARCH_GEN_AI__COST__TOTAL_TOKENS]).toBeGreaterThan(0); + + await expect + .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_GEN_AI__TOOL__NAME]: 'get_weather', + // The model picks the argument (`Paris`, `Paris, France`, ...). + [SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS]: expect.stringContaining('Paris'), + [SEARCH_GEN_AI__TOOL__CALL__RESULT]: expect.stringMatching(/^It is 21 degrees and sunny in Paris/), + }); +}); + +test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { + const conversationId = await runAgentTurn(baseURL!, newInstanceId('manual-span'), 'What is the weather in Berlin?'); + + const { traceId, spans } = await waitForConversationTrace(conversationId, spans => + spans.some( + span => span.description === 'resolve-weather' && spans.some(parent => parent.event_id === span.parent_span_id), + ), + ); + + const manualSpan = spans.find(span => span.description === 'resolve-weather'); + const parent = spans.find(span => span.event_id === manualSpan?.parent_span_id); + expect(parent?.description).toBe('execute_tool get_weather'); + + await expect + .poll(() => fetchSpanAttributes(traceId, manualSpan!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ 'weather.source': 'static-table', 'weather.city': expect.stringContaining('Berlin') }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts new file mode 100644 index 000000000000..7b3f2814266f --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts @@ -0,0 +1,60 @@ +import { expect } from '@playwright/test'; +import { SEARCH_GEN_AI__CONVERSATION__ID } from '@sentry/conventions/attributes/search'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; +import { + EVENT_POLLING_OPTIONS, + fetchTrace, + findTraceIdOfSpan, + flattenTrace, + traceTarget, +} from '@sentry-internal/test-utils/cli'; + +/** A fresh instance id, so an earlier settlement cannot end the wait early. */ +export function newInstanceId(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +/** Send one message and wait for the turn to settle. Returns the conversation id. */ +export async function runAgentTurn(baseURL: string, instanceId: string, message: string): Promise { + const url = `${baseURL}/agents/hello/${instanceId}`; + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ kind: 'user', body: message }), + }); + const body = await res.text(); + expect(res.status, body).toBe(202); + + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + const conversation = (await (await fetch(url)).json()) as { conversationId: string; settlements?: unknown[] }; + if (conversation.settlements?.length) { + return conversation.conversationId; + } + await new Promise(resolve => setTimeout(resolve, 1_000)); + } + + throw new Error(`Flue turn for "${instanceId}" did not settle within 90s`); +} + +/** The router answers `202` before the turn runs, so the trace is found by conversation id. */ +export async function waitForConversationTrace( + conversationId: string, + isComplete: (spans: TraceItem[]) => boolean, +): Promise<{ traceId: string; spans: TraceItem[] }> { + let traceId: string | undefined; + await expect + .poll( + () => (traceId = findTraceIdOfSpan(`${SEARCH_GEN_AI__CONVERSATION__ID}:${conversationId}`)), + EVENT_POLLING_OPTIONS, + ) + .toBeDefined(); + + console.log(`Polling for the agent spans: sentry trace view ${traceTarget(traceId!)}`); + + let spans: TraceItem[] = []; + await expect.poll(() => isComplete((spans = flattenTrace(fetchTrace(traceId!)))), EVENT_POLLING_OPTIONS).toBe(true); + + return { traceId: traceId!, spans }; +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json new file mode 100644 index 000000000000..2cb98b13c9f8 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "types": ["node"], + "strict": true, + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "sentry-init.ts", "flue.config.ts", "vite.config.ts"] +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts new file mode 100644 index 000000000000..a77574b64290 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts @@ -0,0 +1,7 @@ +import { flue } from '@flue/vite'; +import { defineConfig } from 'vite'; + +// As `flue init` scaffolds it. Flue keeps dependencies as bare imports, so orchestrion can still hook them. +export default defineConfig({ + plugins: [flue()], +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts index baf29050d836..8ed0c57e52eb 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -16,11 +16,11 @@ export function Hello() { input: v.object({ city: v.string() }), // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, // so this should nest directly under it rather than landing beside it. - run: ({ city }) => + run: ({ data }) => Sentry.startSpan( - { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': city } }, + { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': data.city } }, () => { - return `It is 21 degrees and sunny in ${city}.`; + return `It is 21 degrees and sunny in ${data.city}.`; }, ), }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts index ccf3e552a87d..a829a0ba282e 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -48,6 +48,31 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn' expect(executeTool?.parent_span_id).toBe(invokeAgent?.span_id); }); +// Two turns: if the context leaked, the second would land in the first request's trace (withastro/flue#813). +test('puts each turn under the request that sent it', async ({ baseURL }) => { + for (const prefix of ['first-request', 'second-request']) { + const conversationId = newConversationId(prefix); + const isRequest = (span: SpanLike & { is_segment?: boolean }) => + span.is_segment === true && + span.name?.startsWith('POST') && + Object.values(span.attributes ?? {}).some( + attribute => typeof attribute?.value === 'string' && attribute.value.includes(conversationId), + ); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isRequest) && hasOps(['gen_ai.invoke_agent'])(spansOfTrace), + ); + + await runAgentTurn(baseURL!, conversationId, 'Say hello.'); + + const spans = await spansPromise; + const request = spans.find(isRequest); + const agents = spans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + expect(agents).toHaveLength(1); + expect(agents[0]?.parent_span_id).toBe(request?.span_id); + } +}); + test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( APP, @@ -61,6 +86,8 @@ test('nests a manual span raised inside a tool under that tool span', async ({ b const manualSpan = spans.find(span => span.name === 'resolve-weather'); expect(manualSpan?.attributes?.['weather.source']?.value).toBe('static-table'); + // The model chooses the argument (`Berlin`, `Berlin, Germany`, ...). + expect(manualSpan?.attributes?.['weather.city']?.value).toEqual(expect.stringContaining('Berlin')); expect(manualSpan?.trace_id).toBe(executeTool?.trace_id); expect(manualSpan?.parent_span_id).toBe(executeTool?.span_id); });