test(e2e): Add node-flue-send-to-sentry test app

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/.gitignore

@@ -0,0 +1,7 @@
+node_modules
+dist
+.flue
+*.tsbuildinfo
+results.junit.xml
+test-results
+playwright-report

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/flue.config.ts

@@ -0,0 +1,5 @@
+import { defineConfig } from '@flue/runtime/config';
+
+export default defineConfig({
+  target: 'node',
+});

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "node-flue-send-to-sentry",
+  "description": "The Flue integration as a user runs it: a built Flue agent server, real tool-calling turns through OpenRouter, and the gen_ai spans read back from a real Sentry project",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "vite build",
+    "start": "NODE_OPTIONS='--import=@sentry/node/import' PORT=3030 node dist/server.mjs",
+    "test": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY playwright test",
+    "clean": "npx rimraf node_modules dist pnpm-lock.yaml",
+    "test:build": "pnpm install && pnpm build",
+    "test:build-latest": "pnpm install && pnpm add @flue/runtime@latest @flue/vite@latest @flue/cli@latest && pnpm build",
+    "test:assert": "pnpm test"
+  },
+  "dependencies": {
+    "@flue/runtime": "2.0.5",
+    "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
+    "dataloader": "^2.2.3",
+    "hono": "^4.7.0",
+    "valibot": "^1.5.0"
+  },
+  "devDependencies": {
+    "@flue/cli": "2.0.5",
+    "@flue/vite": "2.0.5",
+    "@playwright/test": "1.64.1",
+    "@sentry-internal/test-utils": "file:../../packed/sentry-internal-test-utils-packed.tgz",
+    "@sentry/conventions": "0.27.0",
+    "@types/node": "24.x",
+    "sentry": "~0.44.1",
+    "typescript": "~5.9.0",
+    "vite": "^8.0.14"
+  },
+  "engines": {
+    "node": "24.x"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true,
+    "optionalVariants": [
+      {
+        "build-command": "pnpm test:build-latest",
+        "label": "node-flue-send-to-sentry (latest)"
+      }
+    ]
+  }
+}

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/playwright.config.ts

@@ -0,0 +1,24 @@
+import { defineConfig } from '@playwright/test';
+
+const port = 3030;
+
+export default defineConfig({
+  testDir: './tests',
+  // A model turn, then Sentry polls of up to 180s each. Spans become queryable within the first one.
+  timeout: 400_000,
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  // Every test spends most of its time polling Sentry, so run them all at once.
+  workers: '100%',
+  reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+  use: {
+    baseURL: `http://localhost:${port}`,
+  },
+  webServer: {
+    command: 'pnpm start',
+    port,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  },
+});

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/pnpm-workspace.yaml

@@ -0,0 +1,6 @@
+allowBuilds:
+  '@google/genai': false
+  core-js-pure: false
+  esbuild: true
+  protobufjs: false
+minimumReleaseAge: 0

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/sentry-init.ts

@@ -0,0 +1,11 @@
+import { instrument } from '@flue/runtime';
+import * as Sentry from '@sentry/node';
+
+// No tunnel: the spans go to the Sentry project behind `E2E_TEST_DSN`, where the tests read them back.
+Sentry.init({
+  environment: 'qa', // dynamic sampling bias to keep transactions
+  dsn: process.env.E2E_TEST_DSN,
+  tracesSampleRate: 1.0,
+});
+
+instrument(Sentry.createFlueInstrumentation());

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/agents/hello.ts

@@ -0,0 +1,50 @@
+'use agent';
+import { useModel, useTool } from '@flue/runtime';
+import * as Sentry from '@sentry/node';
+import * as v from 'valibot';
+import DataLoader from 'dataloader';
+
+export function Hello() {
+  useModel('openrouter/anthropic/claude-haiku-4.5');
+
+  useTool({
+    name: 'get_weather',
+    description: 'Get the current weather for a city.',
+    input: v.object({ city: v.string() }),
+    // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active,
+    // so this should nest directly under it rather than landing beside it.
+    run: ({ data }) =>
+      Sentry.startSpan(
+        { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': data.city } },
+        () => {
+          return `It is 21 degrees and sunny in ${data.city}.`;
+        },
+      ),
+  });
+
+  // Called from inside a tool on purpose: the dataloader span then lands under `execute_tool` in
+  // the agent's trace, which is what "captured alongside the AI spans" has to mean.
+  useTool({
+    name: 'count_items',
+    description: 'Count items by loading them. Call this when the user asks to count items.',
+    input: v.object({}),
+    // Constructed per execution, like node-eve does: a module-level loader caches its keys, so a
+    // second call would skip the batch function and emit no span.
+    run: async () => {
+      const loader = new DataLoader<number, number>(async keys => keys.map(key => key * 2));
+      const doubled = await Promise.all([loader.load(1), loader.load(2), loader.load(3)]);
+      return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`;
+    },
+  });
+
+  useTool({
+    name: 'fail_now',
+    description: 'Always throws an error. Call this when the user asks to trigger a failure.',
+    input: v.object({}),
+    run: () => {
+      throw new Error('Intentional flue tool failure');
+    },
+  });
+
+  return 'You are a helpful assistant. Use get_weather when asked about weather, count_items when asked to count items, and fail_now when asked to fail.';
+}

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/src/app.ts

@@ -0,0 +1,10 @@
+import '../sentry-init.ts';
+import { createAgentRouter } from '@flue/runtime/routing';
+import { Hono } from 'hono';
+import { Hello } from './agents/hello.ts';
+
+const app = new Hono();
+
+app.route('/agents/hello', createAgentRouter(Hello));
+
+export default app;

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/dataloader.test.ts

@@ -0,0 +1,25 @@
+import { expect, test } from '@playwright/test';
+import { SEARCH_ORIGIN } from '@sentry/conventions/attributes/search';
+import { CACHE_GET } from '@sentry/conventions/op';
+import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli';
+import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils';
+
+// Orchestrion only hooks `dataloader` with the loader preloaded (`--import` in the start script).
+test('sends orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => {
+  const conversationId = await runAgentTurn(
+    baseURL!,
+    newInstanceId('dataloader'),
+    'Please call count_items to count the items.',
+  );
+
+  const { traceId, spans } = await waitForConversationTrace(
+    conversationId,
+    spans =>
+      spans.some(span => span.description === 'execute_tool count_items') && spans.some(span => span.op === CACHE_GET),
+  );
+
+  const dataloaderSpan = spans.find(span => span.op === CACHE_GET);
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, dataloaderSpan!.event_id!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({ [SEARCH_ORIGIN]: 'auto.db.dataloader' });
+});

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/errors.test.ts

@@ -0,0 +1,36 @@
+import { expect, test } from '@playwright/test';
+import { SEARCH_GEN_AI__TOOL__NAME, SEARCH_SPAN__STATUS } from '@sentry/conventions/attributes/search';
+import {
+  EVENT_POLLING_OPTIONS,
+  fetchEvent,
+  fetchSpanAttributes,
+  findErrorInTrace,
+} from '@sentry-internal/test-utils/cli';
+import type { TraceItem } from '@sentry-internal/test-utils/cli';
+import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils';
+
+test('sends an error thrown inside a Flue tool and marks its span errored', async ({ baseURL }) => {
+  const conversationId = await runAgentTurn(
+    baseURL!,
+    newInstanceId('failure'),
+    'Please call fail_now to trigger a failure.',
+  );
+
+  const { traceId, spans } = await waitForConversationTrace(conversationId, spans =>
+    spans.some(span => span.description === 'execute_tool fail_now'),
+  );
+
+  let error: TraceItem | undefined;
+  await expect.poll(() => (error = findErrorInTrace(traceId)), EVENT_POLLING_OPTIONS).toBeDefined();
+  await expect
+    .poll(
+      () => fetchEvent(error!.event_id!)?.entries.find(entry => entry.type === 'exception')?.data.values?.[0],
+      EVENT_POLLING_OPTIONS,
+    )
+    .toMatchObject({ value: 'Intentional flue tool failure', mechanism: { type: 'auto.ai.flue' } });
+
+  const executeTool = spans.find(span => span.description === 'execute_tool fail_now');
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({ [SEARCH_GEN_AI__TOOL__NAME]: 'fail_now', [SEARCH_SPAN__STATUS]: 'error' });
+});

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/flue.test.ts

@@ -0,0 +1,105 @@
+import { expect, test } from '@playwright/test';
+import {
+  SEARCH_GEN_AI__AGENT__NAME,
+  SEARCH_GEN_AI__CONVERSATION__ID,
+  SEARCH_GEN_AI__COST__TOTAL_TOKENS,
+  SEARCH_GEN_AI__INPUT__MESSAGES,
+  SEARCH_GEN_AI__OPERATION__NAME,
+  SEARCH_GEN_AI__PROVIDER__NAME,
+  SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS,
+  SEARCH_GEN_AI__TOOL__CALL__RESULT,
+  SEARCH_GEN_AI__TOOL__NAME,
+  SEARCH_GEN_AI__USAGE__INPUT_TOKENS,
+  SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS,
+  SEARCH_ORIGIN,
+  SEARCH_SPAN__STATUS,
+} from '@sentry/conventions/attributes/search';
+import {
+  GEN_AI_CHAT,
+  GEN_AI_EXECUTE_TOOL,
+  GEN_AI_INVOKE_AGENT,
+  HTTP_CLIENT,
+  HTTP_SERVER,
+} from '@sentry/conventions/op';
+import type { TraceItem } from '@sentry-internal/test-utils/cli';
+import { EVENT_POLLING_OPTIONS, fetchSpanAttributes } from '@sentry-internal/test-utils/cli';
+import { newInstanceId, runAgentTurn, waitForConversationTrace } from './utils';
+
+test('sends the request / invoke_agent / chat / execute_tool hierarchy of a Flue turn', async ({ baseURL }) => {
+  const conversationId = await runAgentTurn(baseURL!, newInstanceId('weather'), 'What is the weather in Paris?');
+
+  // Spans land at different times, so wait for the relationships checked below, not just the ops.
+  const isProviderCall = (span: TraceItem, spans: TraceItem[]) =>
+    span.op === HTTP_CLIENT && spans.some(chat => chat.op === GEN_AI_CHAT && chat.event_id === span.parent_span_id);
+  const { traceId, spans } = await waitForConversationTrace(
+    conversationId,
+    spans =>
+      [HTTP_SERVER, GEN_AI_INVOKE_AGENT, GEN_AI_EXECUTE_TOOL].every(op => spans.some(span => span.op === op)) &&
+      spans.some(span => isProviderCall(span, spans)),
+  );
+
+  const request = spans.find(span => span.op === HTTP_SERVER);
+  const agents = spans.filter(span => span.op === GEN_AI_INVOKE_AGENT);
+  const invokeAgent = agents[0];
+  const chats = spans.filter(span => span.op === GEN_AI_CHAT);
+  const executeTool = spans.find(span => span.description === 'execute_tool get_weather');
+
+  // More than one means other conversations leaked into this trace (withastro/flue#813).
+  expect(agents).toHaveLength(1);
+  expect(invokeAgent?.parent_span_id).toBe(request?.event_id);
+  // A chat span anywhere else would be a provider SDK reporting the same call a second time.
+  expect(chats.map(chat => chat.parent_span_id)).toEqual(chats.map(() => invokeAgent?.event_id));
+  expect(executeTool?.parent_span_id).toBe(invokeAgent?.event_id);
+
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, invokeAgent!.event_id!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({
+      [SEARCH_ORIGIN]: 'auto.ai.flue',
+      [SEARCH_GEN_AI__OPERATION__NAME]: 'invoke_agent',
+      [SEARCH_GEN_AI__AGENT__NAME]: 'Hello',
+      [SEARCH_GEN_AI__CONVERSATION__ID]: conversationId,
+    });
+
+  let chatAttributes: Record<string, unknown> | undefined;
+  await expect
+    .poll(() => (chatAttributes = fetchSpanAttributes(traceId, chats[0]!.event_id!)), EVENT_POLLING_OPTIONS)
+    .toMatchObject({
+      [SEARCH_ORIGIN]: 'auto.ai.flue',
+      [SEARCH_SPAN__STATUS]: 'ok',
+      [SEARCH_GEN_AI__PROVIDER__NAME]: 'openrouter',
+      [SEARCH_GEN_AI__INPUT__MESSAGES]: expect.stringContaining('What is the weather in Paris?'),
+    });
+  // The trace-items API returns integer attributes as strings.
+  expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0);
+  expect(Number(chatAttributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBeGreaterThan(0);
+  // Flue computes cost itself; no provider SDK reports it.
+  expect(chatAttributes![SEARCH_GEN_AI__COST__TOTAL_TOKENS]).toBeGreaterThan(0);
+
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, executeTool!.event_id!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({
+      [SEARCH_ORIGIN]: 'auto.ai.flue',
+      [SEARCH_GEN_AI__TOOL__NAME]: 'get_weather',
+      // The model picks the argument (`Paris`, `Paris, France`, ...).
+      [SEARCH_GEN_AI__TOOL__CALL__ARGUMENTS]: expect.stringContaining('Paris'),
+      [SEARCH_GEN_AI__TOOL__CALL__RESULT]: expect.stringMatching(/^It is 21 degrees and sunny in Paris/),
+    });
+});
+
+test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => {
+  const conversationId = await runAgentTurn(baseURL!, newInstanceId('manual-span'), 'What is the weather in Berlin?');
+
+  const { traceId, spans } = await waitForConversationTrace(conversationId, spans =>
+    spans.some(
+      span => span.description === 'resolve-weather' && spans.some(parent => parent.event_id === span.parent_span_id),
+    ),
+  );
+
+  const manualSpan = spans.find(span => span.description === 'resolve-weather');
+  const parent = spans.find(span => span.event_id === manualSpan?.parent_span_id);
+  expect(parent?.description).toBe('execute_tool get_weather');
+
+  await expect
+    .poll(() => fetchSpanAttributes(traceId, manualSpan!.event_id!), EVENT_POLLING_OPTIONS)
+    .toMatchObject({ 'weather.source': 'static-table', 'weather.city': expect.stringContaining('Berlin') });
+});

dev-packages/e2e-tests/test-applications/node-flue-send-to-sentry/tests/utils.ts

@@ -0,0 +1,60 @@
+import { expect } from '@playwright/test';
+import { SEARCH_GEN_AI__CONVERSATION__ID } from '@sentry/conventions/attributes/search';
+import type { TraceItem } from '@sentry-internal/test-utils/cli';
+import {
+  EVENT_POLLING_OPTIONS,
+  fetchTrace,
+  findTraceIdOfSpan,
+  flattenTrace,
+  traceTarget,
+} from '@sentry-internal/test-utils/cli';
+
+/** A fresh instance id, so an earlier settlement cannot end the wait early. */
+export function newInstanceId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+/** Send one message and wait for the turn to settle. Returns the conversation id. */
+export async function runAgentTurn(baseURL: string, instanceId: string, message: string): Promise<string> {
+  const url = `${baseURL}/agents/hello/${instanceId}`;
+
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ kind: 'user', body: message }),
+  });
+  const body = await res.text();
+  expect(res.status, body).toBe(202);
+
+  const deadline = Date.now() + 90_000;
+  while (Date.now() < deadline) {
+    const conversation = (await (await fetch(url)).json()) as { conversationId: string; settlements?: unknown[] };
+    if (conversation.settlements?.length) {
+      return conversation.conversationId;
+    }
+    await new Promise(resolve => setTimeout(resolve, 1_000));
+  }
+
+  throw new Error(`Flue turn for "${instanceId}" did not settle within 90s`);
+}
+
+/** The router answers `202` before the turn runs, so the trace is found by conversation id. */
+export async function waitForConversationTrace(
+  conversationId: string,
+  isComplete: (spans: TraceItem[]) => boolean,
+): Promise<{ traceId: string; spans: TraceItem[] }> {
+  let traceId: string | undefined;
+  await expect
+    .poll(
+      () => (traceId = findTraceIdOfSpan(`${SEARCH_GEN_AI__CONVERSATION__ID}:${conversationId}`)),
+      EVENT_POLLING_OPTIONS,
+    )
+    .toBeDefined();
+
+  console.log(`Polling for the agent spans: sentry trace view ${traceTarget(traceId!)}`);
+
+  let spans: TraceItem[] = [];
+  await expect.poll(() => isComplete((spans = flattenTrace(fetchTrace(traceId!)))), EVENT_POLLING_OPTIONS).toBe(true);
+
+  return { traceId: traceId!, spans };
+}