From ca4feea959068f2628acd34f0f524dd3e7743d3f Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 20 Oct 2026 12:32:31 +0200 Subject: [PATCH 1/4] test(e2e): Add node-flue-send-to-sentry test app A Flue app on Node that runs real OpenRouter turns and reads the gen_ai spans back from a real Sentry project, with a `latest` variant. Also checks in `node-flue` that each turn sits under the request that sent it, and fixes its weather tool to read `({ data })`, the way Flue 2 passes tool input. Co-Authored-By: Claude Opus 5.5 --- .../node-flue-send-to-sentry/.gitignore | 7 ++ .../node-flue-send-to-sentry/flue.config.ts | 5 ++ .../node-flue-send-to-sentry/package.json | 48 ++++++++++++ .../playwright.config.ts | 24 ++++++ .../pnpm-workspace.yaml | 6 ++ .../node-flue-send-to-sentry/sentry-init.ts | 13 ++++ .../src/agents/hello.ts | 53 ++++++++++++++ .../node-flue-send-to-sentry/src/app.ts | 10 +++ .../tests/dataloader.test.ts | 28 +++++++ .../tests/errors.test.ts | 35 +++++++++ .../tests/flue.test.ts | 73 +++++++++++++++++++ .../node-flue-send-to-sentry/tests/utils.ts | 66 +++++++++++++++++ .../node-flue-send-to-sentry/tsconfig.json | 14 ++++ .../node-flue-send-to-sentry/vite.config.ts | 9 +++ .../node-flue/src/agents/hello.ts | 6 +- .../node-flue/tests/flue.test.ts | 28 +++++++ 16 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json create mode 100644 dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore new file mode 100644 index 000000000000..2685f6ec088a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore @@ -0,0 +1,7 @@ +node_modules +dist +.flue +*.tsbuildinfo +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts new file mode 100644 index 000000000000..a31c8523a9a7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from '@flue/runtime/config'; + +export default defineConfig({ + target: 'node', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json new file mode 100644 index 000000000000..b8a9e4817bd3 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json @@ -0,0 +1,48 @@ +{ + "name": "node-flue-send-to-sentry", + "description": "The Flue integration as a user runs it: a built Flue agent server, real tool-calling turns through OpenRouter, and the gen_ai spans read back from a real Sentry project", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "vite build", + "start": "NODE_OPTIONS='--import=@sentry/node/import' PORT=3030 node dist/server.mjs", + "test": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add @flue/runtime@latest @flue/vite@latest @flue/cli@latest && pnpm build", + "test:assert": "pnpm test" + }, + "dependencies": { + "@flue/runtime": "2.0.5", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "dataloader": "^2.2.3", + "hono": "^4.7.0", + "valibot": "^1.5.0" + }, + "devDependencies": { + "@flue/cli": "2.0.5", + "@flue/vite": "2.0.5", + "@playwright/test": "1.64.1", + "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz", + "@types/node": "24.x", + "sentry": "~0.44.1", + "typescript": "~5.9.0", + "vite": "^8.0.14" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-flue-send-to-sentry (latest)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts new file mode 100644 index 000000000000..652480fbf216 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from '@playwright/test'; + +const port = 3030; + +export default defineConfig({ + testDir: './tests', + // Each test drives a real OpenRouter turn and then waits ~2min until its spans are queryable. + timeout: 300_000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + // Every test spends most of its time polling Sentry, so run them all at once. + workers: '100%', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + use: { + baseURL: `http://localhost:${port}`, + }, + webServer: { + command: 'pnpm start', + port, + stdout: 'pipe', + stderr: 'pipe', + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml new file mode 100644 index 000000000000..0e523a16e6f0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml @@ -0,0 +1,6 @@ +allowBuilds: + '@google/genai': false + core-js-pure: false + esbuild: true + protobufjs: false +minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts new file mode 100644 index 000000000000..f4d804029e34 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts @@ -0,0 +1,13 @@ +import { instrument } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; + +// The setup from the docs: no tunnel, so the spans go to the real Sentry project behind +// `E2E_TEST_DSN`, where the tests read them back. Imported as the first line of `src/app.ts` +// because a Flue app has no framework-owned instrumentation hook to auto-discover. +Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: process.env.E2E_TEST_DSN, + tracesSampleRate: 1.0, +}); + +instrument(Sentry.createFlueInstrumentation()); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts new file mode 100644 index 000000000000..8ed0c57e52eb --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts @@ -0,0 +1,53 @@ +'use agent'; +import { useModel, useTool } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; +import * as v from 'valibot'; +import DataLoader from 'dataloader'; + +// The `'use agent'` directive is how `@flue/vite` finds this module and binds an identity to it at +// build time. That binding is the part a hand-written scenario cannot reproduce, so it is the main +// reason this app exists alongside the node-integration-test suite. +export function Hello() { + useModel('openrouter/anthropic/claude-haiku-4.5'); + + useTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + input: v.object({ city: v.string() }), + // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, + // so this should nest directly under it rather than landing beside it. + run: ({ data }) => + Sentry.startSpan( + { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': data.city } }, + () => { + return `It is 21 degrees and sunny in ${data.city}.`; + }, + ), + }); + + // Called from inside a tool on purpose: the dataloader span then lands under `execute_tool` in + // the agent's trace, which is what "captured alongside the AI spans" has to mean. + useTool({ + name: 'count_items', + description: 'Count items by loading them. Call this when the user asks to count items.', + input: v.object({}), + // Constructed per execution, like node-eve does: a module-level loader caches its keys, so a + // second call would skip the batch function and emit no span. + run: async () => { + const loader = new DataLoader(async keys => keys.map(key => key * 2)); + const doubled = await Promise.all([loader.load(1), loader.load(2), loader.load(3)]); + return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`; + }, + }); + + useTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + input: v.object({}), + run: () => { + throw new Error('Intentional flue tool failure'); + }, + }); + + return 'You are a helpful assistant. Use get_weather when asked about weather, count_items when asked to count items, and fail_now when asked to fail.'; +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts new file mode 100644 index 000000000000..b54a3357e4ca --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts @@ -0,0 +1,10 @@ +import '../sentry-init.ts'; +import { createAgentRouter } from '@flue/runtime/routing'; +import { Hono } from 'hono'; +import { Hello } from './agents/hello.ts'; + +const app = new Hono(); + +app.route('/agents/hello', createAgentRouter(Hello)); + +export default app; diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts new file mode 100644 index 000000000000..62012634195e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from '@playwright/test'; +import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +/** + * `dataloader` is instrumented through orchestrion, a module transform, so it only produces spans + * with the loader registered at process start (`--import=@sentry/node/import` in the start script). + * The loader is called from inside a tool so its span lands in the agent's trace, beside the AI spans. + */ +test('sends orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => { + const conversationId = await runAgentTurn( + baseURL!, + newInstanceId('dataloader'), + 'Please call count_items to count the items.', + ); + + const { traceId, spans } = await waitForConversationTrace( + conversationId, + spans => + spans.some(span => span.description === 'execute_tool count_items') && + spans.some(span => span.op === 'cache.get'), + ); + + const dataloaderSpan = spans.find(span => span.op === 'cache.get'); + await expect + .poll(() => fetchSpanAttributes(traceId, dataloaderSpan!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ origin: 'auto.db.dataloader' }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts new file mode 100644 index 000000000000..89b79bca8b56 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from '@playwright/test'; +import { + EVENT_POLLING_OPTIONS, + fetchEvent, + fetchSpanAttributes, + findErrorInTrace, +} from '@sentry-internal/test-utils/cli'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +test('sends an error thrown inside a Flue tool and marks its span errored', async ({ baseURL }) => { + const conversationId = await runAgentTurn( + baseURL!, + newInstanceId('failure'), + 'Please call fail_now to trigger a failure.', + ); + + const { traceId, spans } = await waitForConversationTrace(conversationId, spans => + spans.some(span => span.description === 'execute_tool fail_now'), + ); + + let error: TraceItem | undefined; + await expect.poll(() => (error = findErrorInTrace(traceId)), EVENT_POLLING_OPTIONS).toBeDefined(); + await expect + .poll( + () => fetchEvent(error!.event_id!)?.entries.find(entry => entry.type === 'exception')?.data.values?.[0], + EVENT_POLLING_OPTIONS, + ) + .toMatchObject({ value: 'Intentional flue tool failure', mechanism: { type: 'auto.ai.flue' } }); + + const executeTool = spans.find(span => span.description === 'execute_tool fail_now'); + await expect + .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ 'gen_ai.tool.name': 'fail_now', 'span.status': 'error' }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts new file mode 100644 index 000000000000..6a26c8c591da --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from '@playwright/test'; +import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; +import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; + +test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue turn', async ({ baseURL }) => { + const conversationId = await runAgentTurn(baseURL!, newInstanceId('weather'), 'What is the weather in Paris?'); + + const { traceId, spans } = await waitForConversationTrace(conversationId, spans => + ['http.server', 'gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool', 'http.client'].every(op => + spans.some(span => span.op === op), + ), + ); + + const request = spans.find(span => span.op === 'http.server'); + const agents = spans.filter(span => span.op === 'gen_ai.invoke_agent'); + const invokeAgent = agents[0]; + const chat = spans.find(span => span.op === 'gen_ai.chat'); + const executeTool = spans.find(span => span.description === 'execute_tool get_weather'); + + // A turn has one agent span. More than one means other conversations' runs leaked into this + // trace (https://github.com/withastro/flue/issues/813). + expect(agents).toHaveLength(1); + // On Node the turn runs inside the request that admitted it, so the agent span hangs off the + // request span instead of starting a trace of its own as it does on Cloudflare. + expect(invokeAgent?.parent_span_id).toBe(request?.event_id); + // Tool and chat spans are siblings under the agent invocation, matching how Flue's own + // OpenTelemetry adapter projects them. + expect(chat?.parent_span_id).toBe(invokeAgent?.event_id); + expect(executeTool?.parent_span_id).toBe(invokeAgent?.event_id); + // The provider's HTTP call is made while the chat span is active. + expect(spans.some(span => span.op === 'http.client' && span.parent_span_id === chat?.event_id)).toBe(true); + + await expect + .poll(() => fetchSpanAttributes(traceId, invokeAgent!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ + origin: 'auto.ai.flue', + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.agent.name': 'Hello', + 'gen_ai.conversation.id': conversationId, + }); + + await expect + .poll(() => fetchSpanAttributes(traceId, chat!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ + origin: 'auto.ai.flue', + 'span.status': 'ok', + 'gen_ai.provider.name': 'openrouter', + 'gen_ai.usage.input_tokens': expect.anything(), + 'gen_ai.usage.output_tokens': expect.anything(), + // Flue computes cost itself; no provider SDK reports it. + 'gen_ai.cost.total_tokens': expect.anything(), + }); + + await expect + .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ origin: 'auto.ai.flue', 'gen_ai.tool.name': 'get_weather' }); +}); + +test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { + const conversationId = await runAgentTurn(baseURL!, newInstanceId('manual-span'), 'What is the weather in Berlin?'); + + const { traceId, spans } = await waitForConversationTrace(conversationId, spans => + spans.some(span => span.description === 'resolve-weather'), + ); + + const manualSpan = spans.find(span => span.description === 'resolve-weather'); + const parent = spans.find(span => span.event_id === manualSpan?.parent_span_id); + expect(parent?.description).toBe('execute_tool get_weather'); + + await expect + .poll(() => fetchSpanAttributes(traceId, manualSpan!.event_id!), EVENT_POLLING_OPTIONS) + .toMatchObject({ 'weather.source': 'static-table', 'weather.city': 'Berlin' }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts new file mode 100644 index 000000000000..7653cb3cf6be --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts @@ -0,0 +1,66 @@ +import { expect } from '@playwright/test'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; +import { + EVENT_POLLING_OPTIONS, + fetchTrace, + findTraceIdOfSpan, + flattenTrace, + traceTarget, +} from '@sentry-internal/test-utils/cli'; + +/** An agent instance id nothing has used yet, so a settled record cannot end the wait early. */ +export function newInstanceId(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +/** + * Run one agent turn over Flue's agent router and wait for it to settle. Returns the id of the + * conversation, which the SDK sets as `gen_ai.conversation.id` on the agent span. + * + * `POST /:id` only admits the work (it returns `202` and the turn runs after), so this reads the + * conversation back until it reports a settlement. + */ +export async function runAgentTurn(baseURL: string, instanceId: string, message: string): Promise { + const url = `${baseURL}/agents/hello/${instanceId}`; + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ kind: 'user', body: message }), + }); + expect(res.status).toBe(202); + await res.text(); + + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + const conversation = (await (await fetch(url)).json()) as { conversationId: string; settlements?: unknown[] }; + if (conversation.settlements?.length) { + return conversation.conversationId; + } + await new Promise(resolve => setTimeout(resolve, 1_000)); + } + + throw new Error(`Flue turn for "${instanceId}" did not settle within 90s`); +} + +/** + * Find the trace of a conversation in Sentry and poll it until `isComplete` holds. The router answers + * the `POST` before the turn runs, so the response carries no trace id to look up; the conversation id + * is the handle the test has. + */ +export async function waitForConversationTrace( + conversationId: string, + isComplete: (spans: TraceItem[]) => boolean, +): Promise<{ traceId: string; spans: TraceItem[] }> { + let traceId: string | undefined; + await expect + .poll(() => (traceId = findTraceIdOfSpan(`gen_ai.conversation.id:${conversationId}`)), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + console.log(`Polling for the agent spans: sentry trace view ${traceTarget(traceId!)}`); + + let spans: TraceItem[] = []; + await expect.poll(() => isComplete((spans = flattenTrace(fetchTrace(traceId!)))), EVENT_POLLING_OPTIONS).toBe(true); + + return { traceId: traceId!, spans }; +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json new file mode 100644 index 000000000000..2cb98b13c9f8 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "types": ["node"], + "strict": true, + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "sentry-init.ts", "flue.config.ts", "vite.config.ts"] +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts new file mode 100644 index 000000000000..2f495966e204 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts @@ -0,0 +1,9 @@ +import { flue } from '@flue/vite'; +import { defineConfig } from 'vite'; + +// Unmodified from what `flue init` scaffolds. No externals config is needed: a Flue node build +// already leaves dependencies as bare specifiers, so orchestrion's module transform still sees them +// as real modules. +export default defineConfig({ + plugins: [flue()], +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts index baf29050d836..8ed0c57e52eb 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -16,11 +16,11 @@ export function Hello() { input: v.object({ city: v.string() }), // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, // so this should nest directly under it rather than landing beside it. - run: ({ city }) => + run: ({ data }) => Sentry.startSpan( - { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': city } }, + { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': data.city } }, () => { - return `It is 21 degrees and sunny in ${city}.`; + return `It is 21 degrees and sunny in ${data.city}.`; }, ), }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts index ccf3e552a87d..cd3523cc153d 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -48,6 +48,33 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn' expect(executeTool?.parent_span_id).toBe(invokeAgent?.span_id); }); +// Flue's Node coordinator runs every turn from one long-lived loop that keeps the async context of +// the first request that started it (https://github.com/withastro/flue/issues/813). Two turns, so +// the second one would land in the first request's trace if that leaked. +test('puts each turn under the request that sent it', async ({ baseURL }) => { + for (const prefix of ['first-request', 'second-request']) { + const conversationId = newConversationId(prefix); + const isRequest = (span: SpanLike & { is_segment?: boolean }) => + span.is_segment === true && + span.name?.startsWith('POST') && + Object.values(span.attributes ?? {}).some( + attribute => typeof attribute?.value === 'string' && attribute.value.includes(conversationId), + ); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isRequest) && hasOps(['gen_ai.invoke_agent'])(spansOfTrace), + ); + + await runAgentTurn(baseURL!, conversationId, 'Say hello.'); + + const spans = await spansPromise; + const request = spans.find(isRequest); + const agents = spans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + expect(agents).toHaveLength(1); + expect(agents[0]?.parent_span_id).toBe(request?.span_id); + } +}); + test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( APP, @@ -61,6 +88,7 @@ test('nests a manual span raised inside a tool under that tool span', async ({ b const manualSpan = spans.find(span => span.name === 'resolve-weather'); expect(manualSpan?.attributes?.['weather.source']?.value).toBe('static-table'); + expect(manualSpan?.attributes?.['weather.city']?.value).toBe('Berlin'); expect(manualSpan?.trace_id).toBe(executeTool?.trace_id); expect(manualSpan?.parent_span_id).toBe(executeTool?.span_id); }); From c062e1fff46bbea0909c9ae774c9a7580380d603 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 20 Oct 2026 13:57:31 +0200 Subject: [PATCH 2/4] test(e2e): Align node-flue-send-to-sentry with the other send-to-sentry apps Use the conventions search aliases and op constants, assert token usage, cost and the tool's arguments and result rather than presence alone, check that every chat span sits under the agent span, put the parent relationships into the wait conditions so the checks cannot run before the spans land, give the polls a 400s timeout, and report the response body when a turn is not admitted. Co-Authored-By: Claude Opus 5.5 --- .../node-flue-send-to-sentry/package.json | 1 + .../playwright.config.ts | 5 +- .../tests/dataloader.test.ts | 9 +- .../tests/errors.test.ts | 3 +- .../tests/flue.test.ts | 87 +++++++++++++------ .../node-flue-send-to-sentry/tests/utils.ts | 10 ++- 6 files changed, 80 insertions(+), 35 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json index b8a9e4817bd3..409652d5c19e 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json @@ -25,6 +25,7 @@ "@flue/vite": "2.0.5", "@playwright/test": "1.64.1", "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz", + "@sentry/conventions": "0.27.0", "@types/node": "24.x", "sentry": "~0.44.1", "typescript": "~5.9.0", diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts index 652480fbf216..f91d2e4499f1 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts @@ -4,8 +4,9 @@ const port = 3030; export default defineConfig({ testDir: './tests', - // Each test drives a real OpenRouter turn and then waits ~2min until its spans are queryable. - timeout: 300_000, + // A real OpenRouter turn, then Sentry polls of up to 180s each (EVENT_POLLING_OPTIONS). Spans take about + // two minutes to become queryable, which falls inside the first poll; the later ones land quickly. + timeout: 400_000, fullyParallel: true, forbidOnly: !!process.env.CI, retries: 0, diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts index 62012634195e..53468ea2cba8 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts @@ -1,4 +1,6 @@ import { expect, test } from '@playwright/test'; +import { SEARCH_ORIGIN } from '@sentry/conventions/attributes/search'; +import { CACHE_GET } from '@sentry/conventions/op'; import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; @@ -17,12 +19,11 @@ test('sends orchestrion-instrumented dataloader spans in the same trace as the A const { traceId, spans } = await waitForConversationTrace( conversationId, spans => - spans.some(span => span.description === 'execute_tool count_items') && - spans.some(span => span.op === 'cache.get'), + spans.some(span => span.description === 'execute_tool count_items') && spans.some(span => span.op === CACHE_GET), ); - const dataloaderSpan = spans.find(span => span.op === 'cache.get'); + const dataloaderSpan = spans.find(span => span.op === CACHE_GET); await expect .poll(() => fetchSpanAttributes(traceId, dataloaderSpan!.event_id!), EVENT_POLLING_OPTIONS) - .toMatchObject({ origin: 'auto.db.dataloader' }); + .toMatchObject({ [SEARCH_ORIGIN]: 'auto.db.dataloader' }); }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts index 89b79bca8b56..aa4666c1a3bf 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts @@ -1,4 +1,5 @@ import { expect, test } from '@playwright/test'; +import { SEARCH_GEN_AI__TOOL__NAME, SEARCH_SPAN__STATUS } from '@sentry/conventions/attributes/search'; import { EVENT_POLLING_OPTIONS, fetchEvent, @@ -31,5 +32,5 @@ test('sends an error thrown inside a Flue tool and marks its span errored', asyn const executeTool = spans.find(span => span.description === 'execute_tool fail_now'); await expect .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) - .toMatchObject({ 'gen_ai.tool.name': 'fail_now', 'span.status': 'error' }); + .toMatchObject({ [SEARCH_GEN_AI__TOOL__NAME]: 'fail_now', [SEARCH_SPAN__STATUS]: 'error' }); }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts index 6a26c8c591da..29908fb00526 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts @@ -1,20 +1,47 @@ import { expect, test } from '@playwright/test'; +import { + SEARCH_GEN_AI__AGENT__NAME, + SEARCH_GEN_AI__CONVERSATION__ID, + SEARCH_GEN_AI__COST__TOTAL_TOKENS, + SEARCH_GEN_AI__INPUT__MESSAGES, + SEARCH_GEN_AI__OPERATION__NAME, + SEARCH_GEN_AI__PROVIDER__NAME, + SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS, + SEARCH_GEN_AI__TOOL__CALL__RESULT, + SEARCH_GEN_AI__TOOL__NAME, + SEARCH_GEN_AI__USAGE__INPUT_TOKENS, + SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, + SEARCH_ORIGIN, + SEARCH_SPAN__STATUS, +} from '@sentry/conventions/attributes/search'; +import { + GEN_AI_CHAT, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, + HTTP_CLIENT, + HTTP_SERVER, +} from '@sentry/conventions/op'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue turn', async ({ baseURL }) => { const conversationId = await runAgentTurn(baseURL!, newInstanceId('weather'), 'What is the weather in Paris?'); - const { traceId, spans } = await waitForConversationTrace(conversationId, spans => - ['http.server', 'gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool', 'http.client'].every(op => - spans.some(span => span.op === op), - ), + // Spans land in Sentry at different times, so wait for the relationships the test checks, not just the ops. + const isProviderCall = (span: TraceItem, spans: TraceItem[]) => + span.op === HTTP_CLIENT && spans.some(chat => chat.op === GEN_AI_CHAT && chat.event_id === span.parent_span_id); + const { traceId, spans } = await waitForConversationTrace( + conversationId, + spans => + [HTTP_SERVER, GEN_AI_INVOKE_AGENT, GEN_AI_EXECUTE_TOOL].every(op => spans.some(span => span.op === op)) && + spans.some(span => isProviderCall(span, spans)), ); - const request = spans.find(span => span.op === 'http.server'); - const agents = spans.filter(span => span.op === 'gen_ai.invoke_agent'); + const request = spans.find(span => span.op === HTTP_SERVER); + const agents = spans.filter(span => span.op === GEN_AI_INVOKE_AGENT); const invokeAgent = agents[0]; - const chat = spans.find(span => span.op === 'gen_ai.chat'); + const chats = spans.filter(span => span.op === GEN_AI_CHAT); const executeTool = spans.find(span => span.description === 'execute_tool get_weather'); // A turn has one agent span. More than one means other conversations' runs leaked into this @@ -24,43 +51,53 @@ test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue // request span instead of starting a trace of its own as it does on Cloudflare. expect(invokeAgent?.parent_span_id).toBe(request?.event_id); // Tool and chat spans are siblings under the agent invocation, matching how Flue's own - // OpenTelemetry adapter projects them. - expect(chat?.parent_span_id).toBe(invokeAgent?.event_id); + // OpenTelemetry adapter projects them. A chat span anywhere else would be a provider SDK + // reporting the same model call a second time. + expect(chats.map(chat => chat.parent_span_id)).toEqual(chats.map(() => invokeAgent?.event_id)); expect(executeTool?.parent_span_id).toBe(invokeAgent?.event_id); - // The provider's HTTP call is made while the chat span is active. - expect(spans.some(span => span.op === 'http.client' && span.parent_span_id === chat?.event_id)).toBe(true); await expect .poll(() => fetchSpanAttributes(traceId, invokeAgent!.event_id!), EVENT_POLLING_OPTIONS) .toMatchObject({ - origin: 'auto.ai.flue', - 'gen_ai.operation.name': 'invoke_agent', - 'gen_ai.agent.name': 'Hello', - 'gen_ai.conversation.id': conversationId, + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_GEN_AI__OPERATION__NAME]: 'invoke_agent', + [SEARCH_GEN_AI__AGENT__NAME]: 'Hello', + [SEARCH_GEN_AI__CONVERSATION__ID]: conversationId, }); + let chatAttributes: Record | undefined; await expect - .poll(() => fetchSpanAttributes(traceId, chat!.event_id!), EVENT_POLLING_OPTIONS) + .poll(() => (chatAttributes = fetchSpanAttributes(traceId, chats[0]!.event_id!)), EVENT_POLLING_OPTIONS) .toMatchObject({ - origin: 'auto.ai.flue', - 'span.status': 'ok', - 'gen_ai.provider.name': 'openrouter', - 'gen_ai.usage.input_tokens': expect.anything(), - 'gen_ai.usage.output_tokens': expect.anything(), - // Flue computes cost itself; no provider SDK reports it. - 'gen_ai.cost.total_tokens': expect.anything(), + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_SPAN__STATUS]: 'ok', + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openrouter', + [SEARCH_GEN_AI__INPUT__MESSAGES]: expect.stringContaining('What is the weather in Paris?'), }); + // The trace-items API returns integer attributes as strings. + expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBeGreaterThan(0); + // Flue computes cost itself; no provider SDK reports it. + expect(chatAttributes![SEARCH_GEN_AI__COST__TOTAL_TOKENS]).toBeGreaterThan(0); await expect .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS) - .toMatchObject({ origin: 'auto.ai.flue', 'gen_ai.tool.name': 'get_weather' }); + .toMatchObject({ + [SEARCH_ORIGIN]: 'auto.ai.flue', + [SEARCH_GEN_AI__TOOL__NAME]: 'get_weather', + // The model chooses the argument (`Paris`, `Paris, France`, ...); the tool's sentence is fixed. + [SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS]: expect.stringContaining('Paris'), + [SEARCH_GEN_AI__TOOL__CALL__RESULT]: expect.stringMatching(/^It is 21 degrees and sunny in Paris/), + }); }); test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { const conversationId = await runAgentTurn(baseURL!, newInstanceId('manual-span'), 'What is the weather in Berlin?'); const { traceId, spans } = await waitForConversationTrace(conversationId, spans => - spans.some(span => span.description === 'resolve-weather'), + spans.some( + span => span.description === 'resolve-weather' && spans.some(parent => parent.event_id === span.parent_span_id), + ), ); const manualSpan = spans.find(span => span.description === 'resolve-weather'); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts index 7653cb3cf6be..e64f0cf3e950 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts @@ -1,4 +1,5 @@ import { expect } from '@playwright/test'; +import { SEARCH_GEN_AI__CONVERSATION__ID } from '@sentry/conventions/attributes/search'; import type { TraceItem } from '@sentry-internal/test-utils/cli'; import { EVENT_POLLING_OPTIONS, @@ -28,8 +29,8 @@ export async function runAgentTurn(baseURL: string, instanceId: string, message: headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ kind: 'user', body: message }), }); - expect(res.status).toBe(202); - await res.text(); + const body = await res.text(); + expect(res.status, body).toBe(202); const deadline = Date.now() + 90_000; while (Date.now() < deadline) { @@ -54,7 +55,10 @@ export async function waitForConversationTrace( ): Promise<{ traceId: string; spans: TraceItem[] }> { let traceId: string | undefined; await expect - .poll(() => (traceId = findTraceIdOfSpan(`gen_ai.conversation.id:${conversationId}`)), EVENT_POLLING_OPTIONS) + .poll( + () => (traceId = findTraceIdOfSpan(`${SEARCH_GEN_AI__CONVERSATION__ID}:${conversationId}`)), + EVENT_POLLING_OPTIONS, + ) .toBeDefined(); console.log(`Polling for the agent spans: sentry trace view ${traceTarget(traceId!)}`); From 74e964b654d06f34304acaf8e67bfb27691558c6 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 20 Oct 2026 14:28:25 +0200 Subject: [PATCH 3/4] test(e2e): Accept any city string the model sends in the Flue manual-span tests The model chooses the `get_weather` argument and may send `Berlin, Germany` as well as `Berlin`, so match on the city name rather than the exact string, as the Paris check already does. Co-Authored-By: Claude Opus 5.5 --- .../node-flue-send-to-sentry/tests/flue.test.ts | 2 +- .../e2e-tests/test-applications/node-flue/tests/flue.test.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts index 29908fb00526..b3ba60bca878 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts @@ -106,5 +106,5 @@ test('nests a manual span raised inside a tool under that tool span', async ({ b await expect .poll(() => fetchSpanAttributes(traceId, manualSpan!.event_id!), EVENT_POLLING_OPTIONS) - .toMatchObject({ 'weather.source': 'static-table', 'weather.city': 'Berlin' }); + .toMatchObject({ 'weather.source': 'static-table', 'weather.city': expect.stringContaining('Berlin') }); }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts index cd3523cc153d..1cf40b9eb96e 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -88,7 +88,8 @@ test('nests a manual span raised inside a tool under that tool span', async ({ b const manualSpan = spans.find(span => span.name === 'resolve-weather'); expect(manualSpan?.attributes?.['weather.source']?.value).toBe('static-table'); - expect(manualSpan?.attributes?.['weather.city']?.value).toBe('Berlin'); + // The model chooses the argument (`Berlin`, `Berlin, Germany`, ...). + expect(manualSpan?.attributes?.['weather.city']?.value).toEqual(expect.stringContaining('Berlin')); expect(manualSpan?.trace_id).toBe(executeTool?.trace_id); expect(manualSpan?.parent_span_id).toBe(executeTool?.span_id); }); From 76bea671c175657d55e66604f1ee31753b4ca075 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 21 Oct 2026 10:37:46 +0200 Subject: [PATCH 4/4] test(e2e): Trim comments in the Flue e2e apps Keep only the reason for each comment, and drop the `'use agent'` explanation from the agent module, as asked in review. Co-Authored-By: Claude Opus 5.5 --- .../playwright.config.ts | 3 +-- .../node-flue-send-to-sentry/sentry-init.ts | 4 +--- .../node-flue-send-to-sentry/src/agents/hello.ts | 3 --- .../tests/dataloader.test.ts | 6 +----- .../node-flue-send-to-sentry/tests/flue.test.ts | 13 ++++--------- .../node-flue-send-to-sentry/tests/utils.ts | 16 +++------------- .../node-flue-send-to-sentry/vite.config.ts | 4 +--- .../node-flue/tests/flue.test.ts | 4 +--- 8 files changed, 12 insertions(+), 41 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts index f91d2e4499f1..161e4b1c5d72 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts @@ -4,8 +4,7 @@ const port = 3030; export default defineConfig({ testDir: './tests', - // A real OpenRouter turn, then Sentry polls of up to 180s each (EVENT_POLLING_OPTIONS). Spans take about - // two minutes to become queryable, which falls inside the first poll; the later ones land quickly. + // A model turn, then Sentry polls of up to 180s each. Spans become queryable within the first one. timeout: 400_000, fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts index f4d804029e34..30f0b4c8cd49 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts @@ -1,9 +1,7 @@ import { instrument } from '@flue/runtime'; import * as Sentry from '@sentry/node'; -// The setup from the docs: no tunnel, so the spans go to the real Sentry project behind -// `E2E_TEST_DSN`, where the tests read them back. Imported as the first line of `src/app.ts` -// because a Flue app has no framework-owned instrumentation hook to auto-discover. +// No tunnel: the spans go to the Sentry project behind `E2E_TEST_DSN`, where the tests read them back. Sentry.init({ environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.E2E_TEST_DSN, diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts index 8ed0c57e52eb..9d218454a86e 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts @@ -4,9 +4,6 @@ import * as Sentry from '@sentry/node'; import * as v from 'valibot'; import DataLoader from 'dataloader'; -// The `'use agent'` directive is how `@flue/vite` finds this module and binds an identity to it at -// build time. That binding is the part a hand-written scenario cannot reproduce, so it is the main -// reason this app exists alongside the node-integration-test suite. export function Hello() { useModel('openrouter/anthropic/claude-haiku-4.5'); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts index 53468ea2cba8..ba6a62fc8f7d 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts @@ -4,11 +4,7 @@ import { CACHE_GET } from '@sentry/conventions/op'; import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli'; import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; -/** - * `dataloader` is instrumented through orchestrion, a module transform, so it only produces spans - * with the loader registered at process start (`--import=@sentry/node/import` in the start script). - * The loader is called from inside a tool so its span lands in the agent's trace, beside the AI spans. - */ +// Orchestrion only hooks `dataloader` with the loader preloaded (`--import` in the start script). test('sends orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => { const conversationId = await runAgentTurn( baseURL!, diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts index b3ba60bca878..6610ad009546 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts @@ -28,7 +28,7 @@ import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils'; test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue turn', async ({ baseURL }) => { const conversationId = await runAgentTurn(baseURL!, newInstanceId('weather'), 'What is the weather in Paris?'); - // Spans land in Sentry at different times, so wait for the relationships the test checks, not just the ops. + // Spans land at different times, so wait for the relationships checked below, not just the ops. const isProviderCall = (span: TraceItem, spans: TraceItem[]) => span.op === HTTP_CLIENT && spans.some(chat => chat.op === GEN_AI_CHAT && chat.event_id === span.parent_span_id); const { traceId, spans } = await waitForConversationTrace( @@ -44,15 +44,10 @@ test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue const chats = spans.filter(span => span.op === GEN_AI_CHAT); const executeTool = spans.find(span => span.description === 'execute_tool get_weather'); - // A turn has one agent span. More than one means other conversations' runs leaked into this - // trace (https://github.com/withastro/flue/issues/813). + // More than one means other conversations leaked into this trace (withastro/flue#813). expect(agents).toHaveLength(1); - // On Node the turn runs inside the request that admitted it, so the agent span hangs off the - // request span instead of starting a trace of its own as it does on Cloudflare. expect(invokeAgent?.parent_span_id).toBe(request?.event_id); - // Tool and chat spans are siblings under the agent invocation, matching how Flue's own - // OpenTelemetry adapter projects them. A chat span anywhere else would be a provider SDK - // reporting the same model call a second time. + // A chat span anywhere else would be a provider SDK reporting the same call a second time. expect(chats.map(chat => chat.parent_span_id)).toEqual(chats.map(() => invokeAgent?.event_id)); expect(executeTool?.parent_span_id).toBe(invokeAgent?.event_id); @@ -85,7 +80,7 @@ test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue .toMatchObject({ [SEARCH_ORIGIN]: 'auto.ai.flue', [SEARCH_GEN_AI__TOOL__NAME]: 'get_weather', - // The model chooses the argument (`Paris`, `Paris, France`, ...); the tool's sentence is fixed. + // The model picks the argument (`Paris`, `Paris, France`, ...). [SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS]: expect.stringContaining('Paris'), [SEARCH_GEN_AI__TOOL__CALL__RESULT]: expect.stringMatching(/^It is 21 degrees and sunny in Paris/), }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts index e64f0cf3e950..7b3f2814266f 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts @@ -9,18 +9,12 @@ import { traceTarget, } from '@sentry-internal/test-utils/cli'; -/** An agent instance id nothing has used yet, so a settled record cannot end the wait early. */ +/** A fresh instance id, so an earlier settlement cannot end the wait early. */ export function newInstanceId(prefix: string): string { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; } -/** - * Run one agent turn over Flue's agent router and wait for it to settle. Returns the id of the - * conversation, which the SDK sets as `gen_ai.conversation.id` on the agent span. - * - * `POST /:id` only admits the work (it returns `202` and the turn runs after), so this reads the - * conversation back until it reports a settlement. - */ +/** Send one message and wait for the turn to settle. Returns the conversation id. */ export async function runAgentTurn(baseURL: string, instanceId: string, message: string): Promise { const url = `${baseURL}/agents/hello/${instanceId}`; @@ -44,11 +38,7 @@ export async function runAgentTurn(baseURL: string, instanceId: string, message: throw new Error(`Flue turn for "${instanceId}" did not settle within 90s`); } -/** - * Find the trace of a conversation in Sentry and poll it until `isComplete` holds. The router answers - * the `POST` before the turn runs, so the response carries no trace id to look up; the conversation id - * is the handle the test has. - */ +/** The router answers `202` before the turn runs, so the trace is found by conversation id. */ export async function waitForConversationTrace( conversationId: string, isComplete: (spans: TraceItem[]) => boolean, diff --git a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts index 2f495966e204..a77574b64290 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/vite.config.ts @@ -1,9 +1,7 @@ import { flue } from '@flue/vite'; import { defineConfig } from 'vite'; -// Unmodified from what `flue init` scaffolds. No externals config is needed: a Flue node build -// already leaves dependencies as bare specifiers, so orchestrion's module transform still sees them -// as real modules. +// As `flue init` scaffolds it. Flue keeps dependencies as bare imports, so orchestrion can still hook them. export default defineConfig({ plugins: [flue()], }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts index 1cf40b9eb96e..a829a0ba282e 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -48,9 +48,7 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn' expect(executeTool?.parent_span_id).toBe(invokeAgent?.span_id); }); -// Flue's Node coordinator runs every turn from one long-lived loop that keeps the async context of -// the first request that started it (https://github.com/withastro/flue/issues/813). Two turns, so -// the second one would land in the first request's trace if that leaked. +// Two turns: if the context leaked, the second would land in the first request's trace (withastro/flue#813). test('puts each turn under the request that sent it', async ({ baseURL }) => { for (const prefix of ['first-request', 'second-request']) { const conversationId = newConversationId(prefix);