From c9fedd8a37fe8aafa5adb245b84d87746069cfb7 Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Wed, 14 Oct 2026 17:09:09 -0400 Subject: [PATCH 1/3] feat(datasetAttributes): provide pagination option + attribute description based evals --- .../src/tools/support/search-events/config.ts | 7 +- .../tools/support/search-events/utils.test.ts | 84 +++++++++++++++++ .../src/tools/support/search-events/utils.ts | 94 +++++++++++++++---- ...rch-events-agent-attribute-context.eval.ts | 45 +++++++++ ...-attributes-spans-number-with-context.json | 14 +++ ...-attributes-spans-string-with-context.json | 24 +++++ 6 files changed, 249 insertions(+), 19 deletions(-) diff --git a/packages/mcp-core/src/tools/support/search-events/config.ts b/packages/mcp-core/src/tools/support/search-events/config.ts index dab182818..0d3194bf0 100644 --- a/packages/mcp-core/src/tools/support/search-events/config.ts +++ b/packages/mcp-core/src/tools/support/search-events/config.ts @@ -41,9 +41,10 @@ TOOL USAGE GUIDELINES: 5. IMPORTANT: For ambiguous terms like "user agents", "browser", "client" - use the appropriate field discovery tool instead of guessing field names 6. When the user already supplied Sentry search syntax for spans/logs/metrics, call datasetAttributes with substringMatch or query filters from the request before dropping or renaming fields 7. If datasetAttributes lists a field under Deprecated Fields, use its replacement instead, even when the deprecated name appears in the guidance or examples in this prompt -8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated -9. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass -10. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query +8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated. If the field you need is not on the first page, call datasetAttributes again with the offset from the end of the response to see more fields +9. When several fields could match the user's request, choose the field whose description best matches it if applicable, not just the closest name +10. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass +11. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query CRITICAL - TOOL RESPONSE HANDLING: All tools return responses in this format: {error?: string, result?: data} diff --git a/packages/mcp-core/src/tools/support/search-events/utils.test.ts b/packages/mcp-core/src/tools/support/search-events/utils.test.ts index e7e84857b..8d8c7e224 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.test.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.test.ts @@ -5,6 +5,7 @@ import type { z } from "zod"; import { SentryApiService } from "../../../api-client"; import * as logging from "../../../telem/logging"; import { + createDatasetAttributesTool, createValidateEventsSearchTool, fetchCustomAttributes, formatEventsValidationResults, @@ -394,6 +395,89 @@ describe("search query helpers", () => { }); }); +describe("datasetAttributes tool pagination", () => { + beforeEach(() => { + mswServer.use( + http.get( + "https://sentry.io/api/0/organizations/test-org/trace-items/attributes/", + () => + HttpResponse.json([ + { + key: "custom.first", + name: "custom.first", + attributeType: "string", + attributeSource: { source_type: "user" }, + }, + { + key: "device.memory", + name: "device.memory", + attributeType: "number", + attributeSource: { source_type: "user" }, + context: { + isConvention: true, + brief: "The estimated total memory capacity of the device.", + isDeprecated: false, + }, + }, + ]), + ), + ); + }); + + afterEach(() => { + mswServer.resetHandlers(); + }); + + async function listAttributes(offset?: number) { + const tool = createDatasetAttributesTool({ + apiService: new SentryApiService({ accessToken: "test-token" }), + organizationSlug: "test-org", + }); + type ToolInput = Parameters>[0]; + const schema = tool.inputSchema as z.ZodType; + const output = await tool.execute!( + schema.parse({ dataset: "spans", offset }), + { + toolCallId: "dataset-attributes-test", + messages: [], + }, + ); + return (output as { result: string }).result; + } + + it("points to the next page when fields are past the first page", async () => { + const result = await listAttributes(); + + expect(result).toMatch(/Available Fields \(\d+ total, showing 1-50\):/); + expect(result).toContain( + "call datasetAttributes again with the same arguments and offset: 50", + ); + expect(result).not.toContain("- device.memory: The estimated"); + expect(result).toContain("EXAMPLE QUERIES FOR SPANS"); + }); + + it("returns later pages with descriptions and types for their fields only", async () => { + const result = await listAttributes(50); + + expect(result).toMatch(/showing 51-\d+\):/); + expect(result).toContain( + "- device.memory: The estimated total memory capacity of the device.", + ); + expect(result).toContain("- device.memory: number"); + expect(result).not.toContain("- span.op:"); + expect(result).not.toContain("offset:"); + expect(result).not.toContain("EXAMPLE QUERIES"); + }); + + it("reports when the offset is past the last field", async () => { + const result = await listAttributes(1000); + + expect(result).toMatch( + /Available Fields \(\d+ total\): none at offset 1000\./, + ); + }); +}); + describe("fetchCustomAttributes", () => { let apiService: SentryApiService; diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index a8d699d7c..d9c7fd96a 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -30,6 +30,7 @@ export type FlexibleEventData = Record; const DEFAULT_MAX_VALUE_LENGTH = 200; const DEFAULT_MAX_ARRAY_ITEMS = 20; +const FIELDS_PAGE_SIZE = 50; const SENTRY_SEARCH_TOKEN_PATTERN = /(^|\s)!?([A-Za-z_][A-Za-z0-9_.[\],-]*):(?=\S)(?!\/\/)/g; const KNOWN_SENTRY_SEARCH_KEYS = new Set([ @@ -1058,8 +1059,22 @@ export function createDatasetAttributesTool(options: { .describe( "Optional attribute types to list. Use ['string','number','boolean'] when unsure.", ), + offset: z + .number() + .int() + .min(0) + .optional() + .describe( + "Optional position in the Available Fields list to start from. Use the offset given at the end of the previous response to see the next page of fields.", + ), }), - execute: async ({ dataset, substringMatch, query, attributeTypes }) => { + execute: async ({ + dataset, + substringMatch, + query, + attributeTypes, + offset = 0, + }) => { const { BASE_COMMON_FIELDS, DATASET_FIELDS, @@ -1088,10 +1103,33 @@ export function createDatasetAttributesTool(options: { }, ); + let commonFields: Record = {}; + let datasetFields: Record = {}; + if (substringMatch) { + Object.entries(BASE_COMMON_FIELDS).forEach(([key, value]) => { + if (key.includes(substringMatch) || value.includes(substringMatch)) { + commonFields[key] = value; + } + }); + Object.entries(DATASET_FIELDS[normalizedDataset]).forEach( + ([key, value]) => { + if ( + key.includes(substringMatch) || + value.includes(substringMatch) + ) { + datasetFields[key] = value; + } + }, + ); + } else { + commonFields = BASE_COMMON_FIELDS; + datasetFields = DATASET_FIELDS[normalizedDataset]; + } + // Combine all available fields const allFields: Record = { - ...toAttributeRecord(BASE_COMMON_FIELDS), - ...toAttributeRecord(DATASET_FIELDS[normalizedDataset]), + ...toAttributeRecord(commonFields), + ...toAttributeRecord(datasetFields), ...customAttributes, }; @@ -1120,14 +1158,45 @@ export function createDatasetAttributesTool(options: { recordAgentToolResultCount(fieldCount); + // Page through fields to avoid overwhelming the agent + const pageFields = [...availableFields].slice( + offset, + offset + FIELDS_PAGE_SIZE, + ); + const pageEnd = offset + pageFields.length; + const fieldsHeader = + pageFields.length > 0 + ? `Available Fields (${fieldCount} total, showing ${offset + 1}-${pageEnd}):` + : `Available Fields (${fieldCount} total):${offset > 0 ? ` none at offset ${offset}.` : ""}`; + const fieldsList = pageFields + .map( + ([key, field]) => + `- ${key} (${allFieldTypes[key]}): ${describeField(field)}`, + ) + .join("\n"); + const nextPageHint = + pageEnd < fieldCount + ? `\n... and ${fieldCount - pageEnd} more fields. To see them, call datasetAttributes again with the same arguments and offset: ${pageEnd}` + : ""; + + // Later pages only list their fields; the deprecated fields, recommended + // fields, and examples were already returned with the first page. + if (offset > 0) { + return `Dataset: ${dataset} + +${fieldsHeader} +${fieldsList} +${nextPageHint} + +IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields. +`; + } + return `Dataset: ${dataset} -Available Fields (${fieldCount} total): -${[...availableFields] - .slice(0, 50) // Limit to first 50 to avoid overwhelming the agent - .map(([key, field]) => `- ${key}: ${describeField(field)}`) - .join("\n")} -${fieldCount > 50 ? `\n... and ${fieldCount - 50} more fields` : ""} +${fieldsHeader} +${fieldsList} +${nextPageHint} ${ replacedFields.length > 0 ? ` @@ -1142,13 +1211,6 @@ ${replacedFields.length > 30 ? `\n... and ${replacedFields.length - 30} more dep Recommended Fields for ${dataset}: ${recommendedFields.basic.map((f) => `- ${f}`).join("\n")} -Field Types (CRITICAL for aggregate functions): -${fieldTypeEntries - .slice(0, 30) // Show more field types since this is critical for aggregate functions - .map(([key, type]) => `- ${key}: ${type}`) - .join("\n")} -${fieldTypeEntries.length > 30 ? `\n... and ${fieldTypeEntries.length - 30} more fields` : ""} - IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields. EXAMPLE QUERIES FOR ${dataset.toUpperCase()}: diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 9adbb774a..908876a90 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -225,6 +225,51 @@ describeEval("search-events-agent-attributes-with-context", { !/(? + Array.isArray(value) && + value.includes("avg(device.memory)") && + !value.some( + (field) => + typeof field === "string" && field.includes("deviceMemory"), + ), + }, + }, + { + // device.model and device.model_id are near-identical names, but only + // device.model_id's brief describes the exact hardware revision + input: + "Count spans grouped by the device's internal hardware revision over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("device.model_id") && + !value.includes("device.model"), + }, + }, ]; }, task: async (input) => { diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json index b44345c04..3dbc9f34d 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json @@ -43,5 +43,19 @@ "key": "custom.db.pool_size", "name": "custom.db.pool_size", "context": {} + }, + { + "key": "deviceMemory", + "name": "deviceMemory", + "context": {} + }, + { + "key": "device.memory", + "name": "device.memory", + "context": { + "isConvention": true, + "brief": "The estimated total memory capacity of the device.", + "isDeprecated": false + } } ] diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json index c058e04c5..bbaa9ed8a 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json @@ -255,5 +255,29 @@ "key": "custom.feature_flag", "name": "custom.feature_flag", "context": {} + }, + { + "key": "device.model", + "name": "device.model", + "context": { + "isConvention": true, + "brief": "The model name of the device.", + "isDeprecated": false, + "examples": [ + "iPhone 15 Pro" + ] + } + }, + { + "key": "device.model_id", + "name": "device.model_id", + "context": { + "isConvention": true, + "brief": "An internal hardware revision to identify the device exactly.", + "isDeprecated": false, + "examples": [ + "iPhone16,1" + ] + } } ] From 9279a6d5f32dcf42037a45ebb8ff3187bd13d052 Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Thu, 15 Oct 2026 11:23:50 -0400 Subject: [PATCH 2/3] adjust description evals --- ...rch-events-agent-attribute-context.eval.ts | 34 ++++++++++++++++--- ...-attributes-spans-string-with-context.json | 16 ++++----- .../trace-items-attributes-spans-string.json | 8 +++++ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 908876a90..82f8f5a80 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -107,6 +107,29 @@ describeEval("search-events-agent-attributes-without-context", { !value.includes("sentry.environment"), }, }, + { + // kiosk.firmware and kiosk.firmware_revision are made-up attributes, so + // the agent has no prior knowledge of either. Without briefs it can + // only match on the name, and kiosk.firmware_revision matches + // "firmware revision" + input: + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("kiosk.firmware_revision") && + !value.includes("kiosk.firmware"), + }, + }, ]; }, task: async (input) => { @@ -250,10 +273,11 @@ describeEval("search-events-agent-attributes-with-context", { }, }, { - // device.model and device.model_id are near-identical names, but only - // device.model_id's brief describes the exact hardware revision + // kiosk.firmware_revision matches "firmware revision" by name, but its + // brief says it is the staged revision. Only kiosk.firmware's brief + // describes the installed firmware input: - "Count spans grouped by the device's internal hardware revision over the last 7 days", + "Count spans grouped by the kiosk's installed firmware revision over the last 7 days", expectedTools: [ { name: "datasetAttributes", @@ -266,8 +290,8 @@ describeEval("search-events-agent-attributes-with-context", { dataset: "spans", fields: (value: unknown) => Array.isArray(value) && - value.includes("device.model_id") && - !value.includes("device.model"), + value.includes("kiosk.firmware") && + !value.includes("kiosk.firmware_revision"), }, }, ]; diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json index bbaa9ed8a..7a0282a9c 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json @@ -257,26 +257,26 @@ "context": {} }, { - "key": "device.model", - "name": "device.model", + "key": "kiosk.firmware", + "name": "kiosk.firmware", "context": { "isConvention": true, - "brief": "The model name of the device.", + "brief": "The firmware revision currently installed and running on the kiosk.", "isDeprecated": false, "examples": [ - "iPhone 15 Pro" + "4.1.3" ] } }, { - "key": "device.model_id", - "name": "device.model_id", + "key": "kiosk.firmware_revision", + "name": "kiosk.firmware_revision", "context": { "isConvention": true, - "brief": "An internal hardware revision to identify the device exactly.", + "brief": "The firmware revision downloaded and staged for the kiosk's next update. It is not installed until the kiosk restarts.", "isDeprecated": false, "examples": [ - "iPhone16,1" + "4.2.0" ] } } diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json index 115773a13..d2dd48d89 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json @@ -106,5 +106,13 @@ { "key": "custom.feature_flag", "name": "Feature Flag" + }, + { + "key": "kiosk.firmware", + "name": "kiosk.firmware" + }, + { + "key": "kiosk.firmware_revision", + "name": "kiosk.firmware_revision" } ] From 0b017ea22b33cf618a0abcd00b4641aa7fb832c7 Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Thu, 15 Oct 2026 12:30:03 -0400 Subject: [PATCH 3/3] cursor fixes + fix failing unit test --- .../mcp-core/src/tools/support/search-events/utils.test.ts | 3 +-- packages/mcp-core/src/tools/support/search-events/utils.ts | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/mcp-core/src/tools/support/search-events/utils.test.ts b/packages/mcp-core/src/tools/support/search-events/utils.test.ts index 9bdb99e9c..39584e979 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.test.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.test.ts @@ -731,9 +731,8 @@ describe("datasetAttributes tool pagination", () => { expect(result).toMatch(/showing 51-\d+\):/); expect(result).toContain( - "- device.memory: The estimated total memory capacity of the device.", + "- device.memory (number): The estimated total memory capacity of the device.", ); - expect(result).toContain("- device.memory: number"); expect(result).not.toContain("- span.op:"); expect(result).not.toContain("offset:"); expect(result).not.toContain("EXAMPLE QUERIES"); diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index ca3749e29..1fd1da226 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -1263,7 +1263,6 @@ export function createDatasetAttributesTool(options: { replaced: replacedFields, updatedFieldTypes, } = resolveAvailableFields(allFields, fieldTypes); - const replacedKeys = new Set(replacedFields.map((field) => field.key)); const fieldCount = availableFields.size; const recommendedFields = RECOMMENDED_FIELDS[normalizedDataset]; @@ -1277,9 +1276,6 @@ export function createDatasetAttributesTool(options: { for (const field of staticNumericFields) { allFieldTypes[field] = "number"; } - const fieldTypeEntries = Object.entries(allFieldTypes).filter( - ([key]) => !replacedKeys.has(key), - ); recordAgentToolResultCount(fieldCount); @@ -1296,7 +1292,7 @@ export function createDatasetAttributesTool(options: { const fieldsList = pageFields .map( ([key, field]) => - `- ${key} (${allFieldTypes[key]}): ${describeField(field)}`, + `- ${key}${allFieldTypes[key] ? ` (${allFieldTypes[key]})` : ""}: ${describeField(field)}`, ) .join("\n"); const nextPageHint =