diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts index 5f793c2c2e54..b4769bda5b9f 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts @@ -51,6 +51,18 @@ export default Sentry.withSentry( return new Response(JSON.stringify(result)); } + if (url.pathname === '/extract-clairic') { + const result = await ai.run('@cf/cloudflare/clairic', { + model: 'clairic', + state: 'Invoice INV-2291 from Northwind Supply. Total due: $1,080.00.', + fields: { + invoice_number: { type: 'text' }, + total: { type: 'number', decimals: 2, unit: 'USD', instructions: 'Amount due' }, + }, + }); + return new Response(JSON.stringify(result)); + } + if (url.pathname === '/stream') { const stream = (await ai.run('@cf/meta/llama-3.1-8b-instruct', { messages: [{ role: 'user', content: 'What is the capital of France?' }], diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts index 0d7e599cca52..0b69b24a63ed 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts @@ -43,6 +43,18 @@ export class MockAi { }; } + if (model === '@cf/cloudflare/clairic') { + return { + model: 'clairic', + answers: { + invoice_number: { type: 'text', span: [8, 16], confidence: 1, value: 'INV-2291' }, + total: { type: 'number', confidence: 0.99, value: 1080 }, + }, + truncated: false, + usage: { state_tokens: 30, schema_tokens: 20, input_tokens: 50, output_tokens: 4, passes: 1 }, + }; + } + if (inputs?.stream === true) { return createSseStream([ '{"response":"The capital "}', diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index aca3281b19c2..53ac07e0c875 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -24,7 +24,10 @@ import { import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { SDK_VERSION } from '@sentry/core'; import { expect, it } from 'vitest'; -import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { + GEN_AI_EXTRACT_OP, + GEN_AI_REQUEST_STREAM_ATTRIBUTE, +} from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { createRunner } from '../../../runner'; import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils'; @@ -230,6 +233,60 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) await runner.completed(); }); +it('traces a Clairic extraction', async ({ signal }) => { + const runner = createRunner(__dirname) + .ignore('event') + .expect(envelope => { + const spans = getSpansFromEnvelope(envelope); + const segmentSpan = spans.find(span => span.is_segment); + + const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.')); + expect(genAiSpans).toHaveLength(1); + + expect(genAiSpans[0]).toEqual( + expect.objectContaining({ + name: 'extract @cf/cloudflare/clairic', + status: 'ok', + is_segment: false, + attributes: { + [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, + [SENTRY_OP]: { value: GEN_AI_EXTRACT_OP, type: 'string' }, + [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, + [GEN_AI_OPERATION_NAME]: { value: 'extract', type: 'string' }, + [GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clairic', type: 'string' }, + [GEN_AI_RESPONSE_MODEL]: { value: 'clairic', type: 'string' }, + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 50, type: 'integer' }, + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 4, type: 'integer' }, + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 54, type: 'integer' }, + // collect only output messages + [GEN_AI_OUTPUT_MESSAGES]: { + type: 'string', + value: JSON.stringify([ + { + answers: { + invoice_number: { type: 'text', span: [8, 16], confidence: 1, value: 'INV-2291' }, + total: { type: 'number', confidence: 0.99, value: 1080 }, + }, + truncated: false, + }, + ]), + }, + [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' }, + [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' }, + [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' }, + [SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' }, + [SENTRY_SDK_NAME]: { value: 'sentry.javascript.cloudflare', type: 'string' }, + [SENTRY_SDK_VERSION]: { value: SDK_VERSION, type: 'string' }, + [SENTRY_ENVIRONMENT]: { value: 'production', type: 'string' }, + }, + }), + ); + }) + .start(signal); + await runner.makeRequest('get', '/extract-clairic'); + await runner.completed(); +}); + // The Workers AI integration deliberately does not call `captureException` itself. // When a `run` call fails, the error must bubble up out of the fetch handler and be // reported by the top-level Cloudflare instrumentation instead — so it shows up in diff --git a/packages/server-utils/src/ai/core/gen-ai-attributes.ts b/packages/server-utils/src/ai/core/gen-ai-attributes.ts index 6b577af8eb4d..a5b99078a6a2 100644 --- a/packages/server-utils/src/ai/core/gen-ai-attributes.ts +++ b/packages/server-utils/src/ai/core/gen-ai-attributes.ts @@ -34,3 +34,9 @@ export const GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE = 'gen_ai.response.stop_reaso * The tool call ID */ export const GEN_AI_TOOL_CALL_ID_ATTRIBUTE = 'gen_ai.tool.call.id'; + +/** + * The span op for extraction of typed fields from a state (e.g. Cloudflare Clairic) + * TODO: Import from `@sentry/conventions/op` once getsentry/sentry-conventions#728 is released + */ +export const GEN_AI_EXTRACT_OP = 'gen_ai.extract'; diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 29db4d2cd136..d01a40ef3f53 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -3,6 +3,7 @@ import { GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_FREQUENCY_PENALTY, GEN_AI_REQUEST_MAX_TOKENS, @@ -11,6 +12,7 @@ import { GEN_AI_REQUEST_TEMPERATURE, GEN_AI_REQUEST_TOP_K, GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, @@ -19,7 +21,7 @@ import { import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_DECIDE } from '@sentry/conventions/op'; import { isObjectLike, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; -import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; +import { GEN_AI_EXTRACT_OP, GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; import { extractSystemInstructions, getEvaluationInputMessages, @@ -32,12 +34,13 @@ export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; import type { WorkersAiInput, WorkersAiOutput } from './types'; -export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide'; +export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide' | 'extract'; export const WORKERS_AI_OPERATION_SPAN_OPS: Record = { chat: GEN_AI_CHAT, embeddings: GEN_AI_EMBEDDINGS, decide: GEN_AI_DECIDE, + extract: GEN_AI_EXTRACT_OP, }; /** @@ -45,6 +48,13 @@ export const WORKERS_AI_OPERATION_SPAN_OPS: Record model.startsWith(prefix))) { return 'decide'; } + if (typeof model === 'string' && EXTRACT_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) { + return 'extract'; + } if (inputs && typeof inputs === 'object') { if ('messages' in inputs || 'prompt' in inputs) { return 'chat'; @@ -123,6 +136,16 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName: return; } + if (operationName === 'extract') { + // Record the request body as passed, but never the image data. + const { model: _model, ...request } = inputs as Record; + if (Array.isArray(request.images)) { + request.images = request.images.map(() => BLOB_SUBSTITUTE); + } + span.setAttribute(GEN_AI_INPUT_MESSAGES, stringify([request])); + return; + } + // Store embeddings input on a separate attribute if (operationName === 'embeddings') { const text = params.text; @@ -166,6 +189,11 @@ export function addResponseAttributes( return; } + if (operationName === 'extract') { + addExtractResponseAttributes(span, result, recordOutputs); + return; + } + if ( !result || typeof result !== 'object' || @@ -200,3 +228,28 @@ export function addResponseAttributes( setOutputMessagesAttribute(span, { responseText, toolCalls }); } } + +function addExtractResponseAttributes(span: Span, body: unknown, recordOutputs: boolean): void { + if (!isObjectLike(body)) { + return; + } + + const { model, usage, ...response } = body; + + if (typeof model === 'string') { + span.setAttribute(GEN_AI_RESPONSE_MODEL, model); + } + + if (isObjectLike(usage)) { + span.setAttributes( + getTokenUsageAttributes( + typeof usage.input_tokens === 'number' ? usage.input_tokens : undefined, + typeof usage.output_tokens === 'number' ? usage.output_tokens : undefined, + ), + ); + } + + if (recordOutputs) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([response])); + } +} diff --git a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts index 85d690e01421..fab753399663 100644 --- a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts +++ b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts @@ -69,6 +69,10 @@ describe('workers-ai utils', () => { expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('decide'); }); + it('returns "extract" for Cloudflare Clairic models', () => { + expect(getOperationName('@cf/cloudflare/clairic', { state: 'Invoice INV-1', fields: {} })).toBe('extract'); + }); + it('does not return "decide" for other TypeSafe models', () => { expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat'); }); @@ -141,6 +145,34 @@ describe('workers-ai utils', () => { }); describe('addRequestAttributes', () => { + it('records extract requests without the model and image data', () => { + const { span, attributes } = createMockSpan(); + const fields = { total: { type: 'number', unit: 'USD' } }; + + addRequestAttributes( + span, + { + model: 'clairic', + state: 'Total: $10', + fields, + images: ['data:image/png;base64,AAAA', 'data:image/png;base64,BBBB'], + reference_date: '2026-10-21', + }, + 'extract', + ); + + expect(attributes).toEqual({ + [GEN_AI_INPUT_MESSAGES]: JSON.stringify([ + { + state: 'Total: $10', + fields, + images: ['[Blob substitute]', '[Blob substitute]'], + reference_date: '2026-10-21', + }, + ]), + }); + }); + it('records messages and extracts system instructions', () => { const { span, attributes } = createMockSpan(); @@ -226,6 +258,31 @@ describe('workers-ai utils', () => { }); }); + it('records extract responses without the model and usage', () => { + const { span, attributes } = createMockSpan(); + const answers = { total: { type: 'number', value: 10, confidence: 0.99 } }; + + addResponseAttributes( + span, + { + model: 'clairic', + answers, + truncated: false, + usage: { state_tokens: 5, input_tokens: 40, output_tokens: 3, passes: 1 }, + }, + true, + 'extract', + ); + + expect(attributes).toEqual({ + [GEN_AI_RESPONSE_MODEL]: 'clairic', + [GEN_AI_USAGE_INPUT_TOKENS]: 40, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [GEN_AI_USAGE_TOTAL_TOKENS]: 43, + [GEN_AI_OUTPUT_MESSAGES]: JSON.stringify([{ answers, truncated: false }]), + }); + }); + it('sets token usage and computes the total from input and output tokens', () => { const { span, attributes } = createMockSpan();