Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ export default Sentry.withSentry(
return new Response(JSON.stringify(result));
}

if (url.pathname === '/extract-clairic') {
const result = await ai.run('@cf/cloudflare/clairic', {
model: 'clairic',
state: 'Invoice INV-2291 from Northwind Supply. Total due: $1,080.00.',
fields: {
invoice_number: { type: 'text' },
total: { type: 'number', decimals: 2, unit: 'USD', instructions: 'Amount due' },
},
});
return new Response(JSON.stringify(result));
}

if (url.pathname === '/stream') {
const stream = (await ai.run('@cf/meta/llama-3.1-8b-instruct', {
messages: [{ role: 'user', content: 'What is the capital of France?' }],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,18 @@ export class MockAi {
};
}

if (model === '@cf/cloudflare/clairic') {
return {
model: 'clairic',
answers: {
invoice_number: { type: 'text', span: [8, 16], confidence: 1, value: 'INV-2291' },
total: { type: 'number', confidence: 0.99, value: 1080 },
},
truncated: false,
usage: { state_tokens: 30, schema_tokens: 20, input_tokens: 50, output_tokens: 4, passes: 1 },
};
}

if (inputs?.stream === true) {
return createSseStream([
'{"response":"The capital "}',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ import {
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
import { SDK_VERSION } from '@sentry/core';
import { expect, it } from 'vitest';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
import {
GEN_AI_EXTRACT_OP,
GEN_AI_REQUEST_STREAM_ATTRIBUTE,
} from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
import { createRunner } from '../../../runner';
import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils';

Expand Down Expand Up @@ -230,6 +233,60 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal })
await runner.completed();
});

it('traces a Clairic extraction', async ({ signal }) => {
const runner = createRunner(__dirname)
.ignore('event')
.expect(envelope => {
const spans = getSpansFromEnvelope(envelope);
const segmentSpan = spans.find(span => span.is_segment);

const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.'));
expect(genAiSpans).toHaveLength(1);

expect(genAiSpans[0]).toEqual(
expect.objectContaining({
name: 'extract @cf/cloudflare/clairic',
status: 'ok',
is_segment: false,
attributes: {
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
[SENTRY_OP]: { value: GEN_AI_EXTRACT_OP, type: 'string' },
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'extract', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clairic', type: 'string' },
[GEN_AI_RESPONSE_MODEL]: { value: 'clairic', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 50, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 4, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 54, type: 'integer' },
// collect only output messages
[GEN_AI_OUTPUT_MESSAGES]: {
type: 'string',
value: JSON.stringify([
{
answers: {
invoice_number: { type: 'text', span: [8, 16], confidence: 1, value: 'INV-2291' },
total: { type: 'number', confidence: 0.99, value: 1080 },
},
truncated: false,
},
]),
},
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
[SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
[SENTRY_SDK_NAME]: { value: 'sentry.javascript.cloudflare', type: 'string' },
[SENTRY_SDK_VERSION]: { value: SDK_VERSION, type: 'string' },
[SENTRY_ENVIRONMENT]: { value: 'production', type: 'string' },
},
}),
);
})
.start(signal);
await runner.makeRequest('get', '/extract-clairic');
await runner.completed();
});

// The Workers AI integration deliberately does not call `captureException` itself.
// When a `run` call fails, the error must bubble up out of the fetch handler and be
// reported by the top-level Cloudflare instrumentation instead — so it shows up in
Expand Down
6 changes: 6 additions & 0 deletions packages/server-utils/src/ai/core/gen-ai-attributes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,9 @@ export const GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE = 'gen_ai.response.stop_reaso
* The tool call ID
*/
export const GEN_AI_TOOL_CALL_ID_ATTRIBUTE = 'gen_ai.tool.call.id';

/**
* The span op for extraction of typed fields from a state (e.g. Cloudflare Clairic)
* TODO: Import from `@sentry/conventions/op` once getsentry/sentry-conventions#728 is released
*/
export const GEN_AI_EXTRACT_OP = 'gen_ai.extract';
57 changes: 55 additions & 2 deletions packages/server-utils/src/ai/workers-ai/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
GEN_AI_EMBEDDINGS_INPUT,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_FREQUENCY_PENALTY,
GEN_AI_REQUEST_MAX_TOKENS,
Expand All @@ -11,6 +12,7 @@ import {
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_REQUEST_TOP_K,
GEN_AI_REQUEST_TOP_P,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
Expand All @@ -19,7 +21,7 @@ import {
import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_DECIDE } from '@sentry/conventions/op';
import { isObjectLike, stringify } from '@sentry/core';
import type { Span, SpanAttributeValue } from '@sentry/core';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes';
import { GEN_AI_EXTRACT_OP, GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes';
import {
extractSystemInstructions,
getEvaluationInputMessages,
Expand All @@ -32,19 +34,27 @@ export { setOutputMessagesAttribute };
import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants';
import type { WorkersAiInput, WorkersAiOutput } from './types';

export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide';
export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide' | 'extract';

export const WORKERS_AI_OPERATION_SPAN_OPS: Record<WorkersAiOperationName, string> = {
chat: GEN_AI_CHAT,
embeddings: GEN_AI_EMBEDDINGS,
decide: GEN_AI_DECIDE,
extract: GEN_AI_EXTRACT_OP,
};

/**
* Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `decide` spans.
*/
const DECIDE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef'];

/**
* Model ID prefixes of the field extraction models (Cloudflare Clairic), traced as `extract` spans.
*/
const EXTRACT_MODEL_PREFIXES = ['@cf/cloudflare/clairic'];

const BLOB_SUBSTITUTE = '[Blob substitute]';

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm is this a pattern we already have? 😅 why not just e.g. [BLOB] or something like this?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's written like that in https://develop.sentry.dev/sdk/telemetry/traces/modules/ai-agents/#tool-data

Binary blobs in the input list should be replaced with the string "[Blob substitute]" in positions where binary data is expected in a given schema. Only binary blobs in positions where binary data is explicitly expected must be redacted. For example, in OpenAI Completions schema, only binary blobs in content blocks with type image_url, input_audio or file should be redacted.

We don't adhere to this yet everywhere. Needs to be a cleanup over all our integrations at some point, probably for v12.


/**
* Determine the gen_ai operation name from the model and inputs passed to `AI.run`.
* Workers AI exposes a single `run` method, so we infer the operation from the model ID and the input shape.
Expand All @@ -53,6 +63,9 @@ export function getOperationName(model: unknown, inputs: unknown): WorkersAiOper
if (typeof model === 'string' && DECIDE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
return 'decide';
}
if (typeof model === 'string' && EXTRACT_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
return 'extract';
}
if (inputs && typeof inputs === 'object') {
if ('messages' in inputs || 'prompt' in inputs) {
return 'chat';
Expand Down Expand Up @@ -123,6 +136,16 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName:
return;
}

if (operationName === 'extract') {
// Record the request body as passed, but never the image data.
const { model: _model, ...request } = inputs as Record<string, unknown>;
if (Array.isArray(request.images)) {
request.images = request.images.map(() => BLOB_SUBSTITUTE);
}
span.setAttribute(GEN_AI_INPUT_MESSAGES, stringify([request]));
return;
}

// Store embeddings input on a separate attribute
if (operationName === 'embeddings') {
const text = params.text;
Expand Down Expand Up @@ -166,6 +189,11 @@ export function addResponseAttributes(
return;
}

if (operationName === 'extract') {
addExtractResponseAttributes(span, result, recordOutputs);
return;
}

if (
!result ||
typeof result !== 'object' ||
Expand Down Expand Up @@ -200,3 +228,28 @@ export function addResponseAttributes(
setOutputMessagesAttribute(span, { responseText, toolCalls });
}
}

function addExtractResponseAttributes(span: Span, body: unknown, recordOutputs: boolean): void {
if (!isObjectLike(body)) {
return;
}

const { model, usage, ...response } = body;

if (typeof model === 'string') {
span.setAttribute(GEN_AI_RESPONSE_MODEL, model);
}

if (isObjectLike(usage)) {
span.setAttributes(
getTokenUsageAttributes(
typeof usage.input_tokens === 'number' ? usage.input_tokens : undefined,
typeof usage.output_tokens === 'number' ? usage.output_tokens : undefined,
),
);
}

if (recordOutputs) {
span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([response]));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ describe('workers-ai utils', () => {
expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('decide');
});

it('returns "extract" for Cloudflare Clairic models', () => {
expect(getOperationName('@cf/cloudflare/clairic', { state: 'Invoice INV-1', fields: {} })).toBe('extract');
});
Comment thread
cursor[bot] marked this conversation as resolved.

it('does not return "decide" for other TypeSafe models', () => {
expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat');
});
Expand Down Expand Up @@ -141,6 +145,34 @@ describe('workers-ai utils', () => {
});

describe('addRequestAttributes', () => {
it('records extract requests without the model and image data', () => {
const { span, attributes } = createMockSpan();
const fields = { total: { type: 'number', unit: 'USD' } };

addRequestAttributes(
span,
{
model: 'clairic',
state: 'Total: $10',
fields,
images: ['data:image/png;base64,AAAA', 'data:image/png;base64,BBBB'],
reference_date: '2026-10-21',
},
'extract',
);

expect(attributes).toEqual({
[GEN_AI_INPUT_MESSAGES]: JSON.stringify([
{
state: 'Total: $10',
fields,
images: ['[Blob substitute]', '[Blob substitute]'],
reference_date: '2026-10-21',
},
]),
});
});

it('records messages and extracts system instructions', () => {
const { span, attributes } = createMockSpan();

Expand Down Expand Up @@ -226,6 +258,31 @@ describe('workers-ai utils', () => {
});
});

it('records extract responses without the model and usage', () => {
const { span, attributes } = createMockSpan();
const answers = { total: { type: 'number', value: 10, confidence: 0.99 } };

addResponseAttributes(
span,
{
model: 'clairic',
answers,
truncated: false,
usage: { state_tokens: 5, input_tokens: 40, output_tokens: 3, passes: 1 },
},
true,
'extract',
);

expect(attributes).toEqual({
[GEN_AI_RESPONSE_MODEL]: 'clairic',
[GEN_AI_USAGE_INPUT_TOKENS]: 40,
[GEN_AI_USAGE_OUTPUT_TOKENS]: 3,
[GEN_AI_USAGE_TOTAL_TOKENS]: 43,
[GEN_AI_OUTPUT_MESSAGES]: JSON.stringify([{ answers, truncated: false }]),
});
});

it('sets token usage and computes the total from input and output tokens', () => {
const { span, attributes } = createMockSpan();

Expand Down
Loading