diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index dc3df6abb..f095c646a 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -123,11 +123,11 @@ Uses the official MCP SDK (`@modelcontextprotocol/sdk`) to ensure compatibility: ```typescript const server = new Server({ name: "sentry-mcp", - version: "1.0.0" + version: "1.0.0", }); server.setRequestHandler(ListToolsRequestSchema, () => ({ - tools: TOOL_DEFINITIONS + tools: TOOL_DEFINITIONS, })); ``` @@ -176,13 +176,13 @@ Centralized client with method-specific implementations: ```typescript class SentryApiService { - constructor(private config: { host: string; accessToken: string }) - + constructor(private config: { host: string; accessToken: string }); + // Resource-specific methods issues = { list: (params) => this.fetch("/issues/", params), - get: (params) => this.fetch(`/issues/${id}/`) - } + get: (params) => this.fetch(`/issues/${id}/`), + }; } ``` @@ -294,12 +294,14 @@ Execute actions and retrieve data: Some tools (the dataset search tools like `search_errors`/`search_traces`, and `search_issues`) implement a two-tier agent pattern: ### Tier 1: Calling Agent (Claude/Cursor) + - Decides when to use search tools - Provides natural language queries - Handles errors and retries - Interprets results for the user ### Tier 2: Embedded Agent (GPT-5) + - Lives inside the MCP tool - Translates natural language to Sentry query syntax - Has its own tools for field discovery @@ -329,6 +331,7 @@ Some tools (the dataset search tools like `search_errors`/`search_traces`, and ` ### Design Rationale This pattern is used when: + - Query languages are complex (Sentry's search syntax) - Available fields vary by context (project-specific attributes) - Semantic understanding is required ("yesterday" → timeRange) diff --git a/packages/mcp-core/src/api-client/client.ts b/packages/mcp-core/src/api-client/client.ts index a4946e1ae..8e3ef524d 100644 --- a/packages/mcp-core/src/api-client/client.ts +++ b/packages/mcp-core/src/api-client/client.ts @@ -416,7 +416,12 @@ export type EventsValidationResult = { orderby: EventsAttributeValidationResult[]; }; -const TraceItemAttributeTypeSchema = z.enum(["string", "number", "boolean", "array"]); +const TraceItemAttributeTypeSchema = z.enum([ + "string", + "number", + "boolean", + "array", +]); const TraceItemAttributeSourceSchema = z.object({ source_type: z.enum(["sentry", "user"]), is_transformed_alias: z.boolean().optional(), diff --git a/packages/mcp-core/src/tools/support/search-events/config.ts b/packages/mcp-core/src/tools/support/search-events/config.ts index c45aef3fc..94e866f48 100644 --- a/packages/mcp-core/src/tools/support/search-events/config.ts +++ b/packages/mcp-core/src/tools/support/search-events/config.ts @@ -41,9 +41,10 @@ TOOL USAGE GUIDELINES: 5. IMPORTANT: For ambiguous terms like "user agents", "browser", "client" - use the appropriate field discovery tool instead of guessing field names 6. When the user already supplied Sentry search syntax for spans/logs/metrics, call datasetAttributes with substringMatch or query filters from the request before dropping or renaming fields 7. If datasetAttributes lists a field under Deprecated Fields, use its replacement instead, even when the deprecated name appears in the guidance or examples in this prompt -8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated -9. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass -10. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query +8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated. If the field you need is not on the first page, call datasetAttributes again with the offset from the end of the response to see more fields +9. When several fields could match the user's request, choose the field whose description best matches it if applicable, not just the closest name +10. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass +11. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query CRITICAL - TOOL RESPONSE HANDLING: All tools return responses in this format: {error?: string, result?: data} diff --git a/packages/mcp-core/src/tools/support/search-events/utils.test.ts b/packages/mcp-core/src/tools/support/search-events/utils.test.ts index 23ee77616..39584e979 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.test.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.test.ts @@ -5,6 +5,7 @@ import type { z } from "zod"; import { SentryApiService } from "../../../api-client"; import * as logging from "../../../telem/logging"; import { + createDatasetAttributesTool, createValidateEventsSearchTool, fetchCustomAttributes, formatEventsValidationResults, @@ -664,6 +665,88 @@ describe("search query helpers", () => { }); }); +describe("datasetAttributes tool pagination", () => { + beforeEach(() => { + mswServer.use( + http.get( + "https://sentry.io/api/0/organizations/test-org/trace-items/attributes/", + () => + HttpResponse.json([ + { + key: "custom.first", + name: "custom.first", + attributeType: "string", + attributeSource: { source_type: "user" }, + }, + { + key: "device.memory", + name: "device.memory", + attributeType: "number", + attributeSource: { source_type: "user" }, + context: { + isConvention: true, + brief: "The estimated total memory capacity of the device.", + isDeprecated: false, + }, + }, + ]), + ), + ); + }); + + afterEach(() => { + mswServer.resetHandlers(); + }); + + async function listAttributes(offset?: number) { + const tool = createDatasetAttributesTool({ + apiService: new SentryApiService({ accessToken: "test-token" }), + organizationSlug: "test-org", + }); + type ToolInput = Parameters>[0]; + const schema = tool.inputSchema as z.ZodType; + const output = await tool.execute!( + schema.parse({ dataset: "spans", offset }), + { + toolCallId: "dataset-attributes-test", + messages: [], + }, + ); + return (output as { result: string }).result; + } + + it("points to the next page when fields are past the first page", async () => { + const result = await listAttributes(); + + expect(result).toMatch(/Available Fields \(\d+ total, showing 1-50\):/); + expect(result).toContain( + "call datasetAttributes again with the same arguments and offset: 50", + ); + expect(result).not.toContain("- device.memory: The estimated"); + expect(result).toContain("EXAMPLE QUERIES FOR SPANS"); + }); + + it("returns later pages with descriptions and types for their fields only", async () => { + const result = await listAttributes(50); + + expect(result).toMatch(/showing 51-\d+\):/); + expect(result).toContain( + "- device.memory (number): The estimated total memory capacity of the device.", + ); + expect(result).not.toContain("- span.op:"); + expect(result).not.toContain("offset:"); + expect(result).not.toContain("EXAMPLE QUERIES"); + }); + + it("reports when the offset is past the last field", async () => { + const result = await listAttributes(1000); + + expect(result).toMatch( + /Available Fields \(\d+ total\): none at offset 1000\./, + ); + }); +}); + describe("fetchCustomAttributes", () => { let apiService: SentryApiService; diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index f04aa58f1..1fd1da226 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -30,6 +30,7 @@ export type FlexibleEventData = Record; const DEFAULT_MAX_VALUE_LENGTH = 200; const DEFAULT_MAX_ARRAY_ITEMS = 20; +const FIELDS_PAGE_SIZE = 50; // Mirrors regex_value in Sentry's event_search.py grammar, which is what // separates a key://pattern// filter from prose like http://example.com. const REGEX_FILTER_VALUE_SOURCE = String.raw`\/\/(?!\/\/(?:[\t\n )]|$))[^\n]{1,1024}?\/\/(?=[\t\n )]|$)`; @@ -1148,7 +1149,12 @@ export function createDatasetAttributesTool(options: { projectId?: string; }) { const { apiService, organizationSlug, projectId } = options; - const traceItemAttributeTypeSchema = z.enum(["string", "number", "boolean", "array"]); + const traceItemAttributeTypeSchema = z.enum([ + "string", + "number", + "boolean", + "array", + ]); return agentTool({ description: @@ -1178,8 +1184,22 @@ export function createDatasetAttributesTool(options: { .describe( "Optional attribute types to list. Use ['string','number','boolean'] when unsure.", ), + offset: z + .number() + .int() + .min(0) + .optional() + .describe( + "Optional position in the Available Fields list to start from. Use the offset given at the end of the previous response to see the next page of fields.", + ), }), - execute: async ({ dataset, substringMatch, query, attributeTypes }) => { + execute: async ({ + dataset, + substringMatch, + query, + attributeTypes, + offset = 0, + }) => { const { BASE_COMMON_FIELDS, DATASET_FIELDS, @@ -1208,10 +1228,33 @@ export function createDatasetAttributesTool(options: { }, ); + let commonFields: Record = {}; + let datasetFields: Record = {}; + if (substringMatch) { + Object.entries(BASE_COMMON_FIELDS).forEach(([key, value]) => { + if (key.includes(substringMatch) || value.includes(substringMatch)) { + commonFields[key] = value; + } + }); + Object.entries(DATASET_FIELDS[normalizedDataset]).forEach( + ([key, value]) => { + if ( + key.includes(substringMatch) || + value.includes(substringMatch) + ) { + datasetFields[key] = value; + } + }, + ); + } else { + commonFields = BASE_COMMON_FIELDS; + datasetFields = DATASET_FIELDS[normalizedDataset]; + } + // Combine all available fields const allFields: Record = { - ...toAttributeRecord(BASE_COMMON_FIELDS), - ...toAttributeRecord(DATASET_FIELDS[normalizedDataset]), + ...toAttributeRecord(commonFields), + ...toAttributeRecord(datasetFields), ...customAttributes, }; @@ -1220,7 +1263,6 @@ export function createDatasetAttributesTool(options: { replaced: replacedFields, updatedFieldTypes, } = resolveAvailableFields(allFields, fieldTypes); - const replacedKeys = new Set(replacedFields.map((field) => field.key)); const fieldCount = availableFields.size; const recommendedFields = RECOMMENDED_FIELDS[normalizedDataset]; @@ -1234,20 +1276,48 @@ export function createDatasetAttributesTool(options: { for (const field of staticNumericFields) { allFieldTypes[field] = "number"; } - const fieldTypeEntries = Object.entries(allFieldTypes).filter( - ([key]) => !replacedKeys.has(key), - ); recordAgentToolResultCount(fieldCount); + // Page through fields to avoid overwhelming the agent + const pageFields = [...availableFields].slice( + offset, + offset + FIELDS_PAGE_SIZE, + ); + const pageEnd = offset + pageFields.length; + const fieldsHeader = + pageFields.length > 0 + ? `Available Fields (${fieldCount} total, showing ${offset + 1}-${pageEnd}):` + : `Available Fields (${fieldCount} total):${offset > 0 ? ` none at offset ${offset}.` : ""}`; + const fieldsList = pageFields + .map( + ([key, field]) => + `- ${key}${allFieldTypes[key] ? ` (${allFieldTypes[key]})` : ""}: ${describeField(field)}`, + ) + .join("\n"); + const nextPageHint = + pageEnd < fieldCount + ? `\n... and ${fieldCount - pageEnd} more fields. To see them, call datasetAttributes again with the same arguments and offset: ${pageEnd}` + : ""; + + // Later pages only list their fields; the deprecated fields, recommended + // fields, and examples were already returned with the first page. + if (offset > 0) { + return `Dataset: ${dataset} + +${fieldsHeader} +${fieldsList} +${nextPageHint} + +IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields. +`; + } + return `Dataset: ${dataset} -Available Fields (${fieldCount} total): -${[...availableFields] - .slice(0, 50) // Limit to first 50 to avoid overwhelming the agent - .map(([key, field]) => `- ${key}: ${describeField(field)}`) - .join("\n")} -${fieldCount > 50 ? `\n... and ${fieldCount - 50} more fields` : ""} +${fieldsHeader} +${fieldsList} +${nextPageHint} ${ replacedFields.length > 0 ? ` @@ -1262,13 +1332,6 @@ ${replacedFields.length > 30 ? `\n... and ${replacedFields.length - 30} more dep Recommended Fields for ${dataset}: ${recommendedFields.basic.map((f) => `- ${f}`).join("\n")} -Field Types (CRITICAL for aggregate functions): -${fieldTypeEntries - .slice(0, 30) // Show more field types since this is critical for aggregate functions - .map(([key, type]) => `- ${key}: ${type}`) - .join("\n")} -${fieldTypeEntries.length > 30 ? `\n... and ${fieldTypeEntries.length - 30} more fields` : ""} - IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields. EXAMPLE QUERIES FOR ${dataset.toUpperCase()}: diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 9adbb774a..82f8f5a80 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -107,6 +107,29 @@ describeEval("search-events-agent-attributes-without-context", { !value.includes("sentry.environment"), }, }, + { + // kiosk.firmware and kiosk.firmware_revision are made-up attributes, so + // the agent has no prior knowledge of either. Without briefs it can + // only match on the name, and kiosk.firmware_revision matches + // "firmware revision" + input: + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("kiosk.firmware_revision") && + !value.includes("kiosk.firmware"), + }, + }, ]; }, task: async (input) => { @@ -225,6 +248,52 @@ describeEval("search-events-agent-attributes-with-context", { !/(? + Array.isArray(value) && + value.includes("avg(device.memory)") && + !value.some( + (field) => + typeof field === "string" && field.includes("deviceMemory"), + ), + }, + }, + { + // kiosk.firmware_revision matches "firmware revision" by name, but its + // brief says it is the staged revision. Only kiosk.firmware's brief + // describes the installed firmware + input: + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("kiosk.firmware") && + !value.includes("kiosk.firmware_revision"), + }, + }, ]; }, task: async (input) => { diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json index b44345c04..3dbc9f34d 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json @@ -43,5 +43,19 @@ "key": "custom.db.pool_size", "name": "custom.db.pool_size", "context": {} + }, + { + "key": "deviceMemory", + "name": "deviceMemory", + "context": {} + }, + { + "key": "device.memory", + "name": "device.memory", + "context": { + "isConvention": true, + "brief": "The estimated total memory capacity of the device.", + "isDeprecated": false + } } ] diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json index c058e04c5..7a0282a9c 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json @@ -255,5 +255,29 @@ "key": "custom.feature_flag", "name": "custom.feature_flag", "context": {} + }, + { + "key": "kiosk.firmware", + "name": "kiosk.firmware", + "context": { + "isConvention": true, + "brief": "The firmware revision currently installed and running on the kiosk.", + "isDeprecated": false, + "examples": [ + "4.1.3" + ] + } + }, + { + "key": "kiosk.firmware_revision", + "name": "kiosk.firmware_revision", + "context": { + "isConvention": true, + "brief": "The firmware revision downloaded and staged for the kiosk's next update. It is not installed until the kiosk restarts.", + "isDeprecated": false, + "examples": [ + "4.2.0" + ] + } } ] diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json index 115773a13..d2dd48d89 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json @@ -106,5 +106,13 @@ { "key": "custom.feature_flag", "name": "Feature Flag" + }, + { + "key": "kiosk.firmware", + "name": "kiosk.firmware" + }, + { + "key": "kiosk.firmware_revision", + "name": "kiosk.firmware_revision" } ]