diff --git a/apps/api/src/chat/loop/turn.ts b/apps/api/src/chat/loop/turn.ts index 0c129c29b..0fd5dd8d0 100644 --- a/apps/api/src/chat/loop/turn.ts +++ b/apps/api/src/chat/loop/turn.ts @@ -341,6 +341,9 @@ const runStep = ( ? annotateModelCallEnd(collected) : Effect.void, ), + // Mapped before the span closes so it records `@maple/llm/LlmCallError` + // with a reason, not the package's own `AI.Error` tag. + Stream.mapError((error) => toLlmCallError("chat.turn", error)), // One span per model call — an attempt, not a logical step, because a // retry costs the same money and wall clock and deserves its own record. // The catch below sits outside, so a failed call ends this span with the @@ -368,9 +371,8 @@ const runStep = ( // A model failure either retries the step or ends the turn as a recorded event. Either // way it does not kill the stream: the session log is durable, so a client reconnecting // after the failure must still be able to read what happened. - Stream.catch((error) => { + Stream.catch((called) => { failed = true - const called = toLlmCallError("chat.turn", error) // Aborted mid-stream. The DO already recorded the terminal event when it cleared the // claim, so emitting anything here would be a second one. diff --git a/apps/api/src/chat/turn-runner.ts b/apps/api/src/chat/turn-runner.ts index 9fbde6db5..05fe19cb0 100644 --- a/apps/api/src/chat/turn-runner.ts +++ b/apps/api/src/chat/turn-runner.ts @@ -154,7 +154,9 @@ const compactIfNeeded = ( usage: TurnUsage, ): Effect.Effect => Effect.gen(function* () { - const { contextLimitOf, outputLimitOf } = yield* Effect.promise(() => import("../platform/Llm")) + const { contextLimitOf, outputLimitOf, toLlmCallError } = yield* Effect.promise( + () => import("../platform/Llm"), + ) const { addUsage, isNearContextLimit, @@ -190,6 +192,8 @@ const compactIfNeeded = ( }) const response = yield* LLM.generate(request).pipe( Effect.tap((generated) => annotateModelResponse(generated)), + // Mapped inside the span so a failed compaction records Maple's tag, not `AI.Error`. + Effect.mapError((error) => toLlmCallError("chat.compaction", error)), Effect.withSpan(modelCallSpanName(model), { kind: "client", // The whole request, so the span records what the model was actually diff --git a/apps/api/src/mcp/lib/dashboard-schema-doc.ts b/apps/api/src/mcp/lib/dashboard-schema-doc.ts index 17b69d5de..0d595c6bb 100644 --- a/apps/api/src/mcp/lib/dashboard-schema-doc.ts +++ b/apps/api/src/mcp/lib/dashboard-schema-doc.ts @@ -500,8 +500,8 @@ const rawSqlSection = (): string => "", "```sql", "SELECT toStartOfInterval(Timestamp, INTERVAL $__interval_s SECOND) AS bucket,", - " countIf(SeverityText = 'Error') AS Error,", - " countIf(SeverityText = 'Warn') AS Warn", + " countIf(SeverityNumber BETWEEN 17 AND 20) AS Error,", + " countIf(SeverityNumber BETWEEN 13 AND 16) AS Warn", "FROM logs", "WHERE $__orgFilter AND $__timeFilter(Timestamp)", "GROUP BY bucket", diff --git a/apps/api/src/mcp/lib/render-trace.test.ts b/apps/api/src/mcp/lib/render-trace.test.ts index e4602b5e1..75063c198 100644 --- a/apps/api/src/mcp/lib/render-trace.test.ts +++ b/apps/api/src/mcp/lib/render-trace.test.ts @@ -98,3 +98,25 @@ describe("renderTraceOverview", () => { expect(text).toContain("span:deadbeef") // short span ref for the error log }) }) + +describe("renderTraceOverview errorsOnly", () => { + it("says which policy pruned the tree", () => { + const spans = [ + span("root", { + children: [span("ok"), span("bad", { statusCode: "Error", statusMessage: "boom" })], + }), + ] + const { lines, overview } = renderTraceOverview({ + ...base, + spanCount: 3, + spans, + budget: 100, + options: { errorsOnly: true }, + }) + const text = lines.join("\n") + expect(overview.renderedCount).toBe(2) + expect(text).toContain("error spans and their ancestors only") + expect(text).toContain("bad") + expect(text).not.toContain("ok —") + }) +}) diff --git a/apps/api/src/mcp/lib/render-trace.ts b/apps/api/src/mcp/lib/render-trace.ts index b49f4ac1c..6173f12e0 100644 --- a/apps/api/src/mcp/lib/render-trace.ts +++ b/apps/api/src/mcp/lib/render-trace.ts @@ -1,7 +1,7 @@ import { Array as Arr, pipe } from "effect" import type { SpanNode } from "@maple/query-engine/observability" import { formatDurationFromMs, truncate } from "./format" -import { selectOverviewSpans, type OverviewSelection } from "./span-tree" +import { selectOverviewSpans, type OverviewOptions, type OverviewSelection } from "./span-tree" export interface TraceOverviewLog { readonly timestamp: string @@ -20,6 +20,7 @@ export interface TraceOverviewInput { readonly logs: ReadonlyArray /** Max spans to render before collapsing the rest (see `selectOverviewSpans`). */ readonly budget: number + readonly options?: OverviewOptions } export interface RenderedTraceOverview { @@ -33,7 +34,7 @@ export interface RenderedTraceOverview { * collapse markers — is unit-testable without a live warehouse. */ export function renderTraceOverview(input: TraceOverviewInput): RenderedTraceOverview { - const overview = selectOverviewSpans(input.spans, input.budget) + const overview = selectOverviewSpans(input.spans, input.budget, input.options) const lines: string[] = [ `## Trace ${input.traceId} (${input.serviceCount} services, ${input.spanCount} spans, ${formatDurationFromMs(input.rootDurationMs)})`, @@ -41,8 +42,11 @@ export function renderTraceOverview(input: TraceOverviewInput): RenderedTraceOve ] if (overview.truncated) { + const policy = input.options?.errorsOnly + ? "error spans and their ancestors only" + : "errors and longest first" lines.push( - `_Showing ${overview.renderedCount} of ${input.spanCount} spans (errors and longest first). Use \`inspect_span trace_id="${input.traceId}" span_id="…"\` for one span's full attributes, or \`search_traces\` to find more._`, + `_Showing ${overview.renderedCount} of ${input.spanCount} spans (${policy}). Use \`inspect_span trace_id="${input.traceId}" span_id="…"\` for one span's full attributes, or \`search_traces\` to find more._`, ``, ) } diff --git a/apps/api/src/mcp/lib/span-tree.test.ts b/apps/api/src/mcp/lib/span-tree.test.ts index 59de998f4..e8563880c 100644 --- a/apps/api/src/mcp/lib/span-tree.test.ts +++ b/apps/api/src/mcp/lib/span-tree.test.ts @@ -119,3 +119,20 @@ describe("selectOverviewSpans", () => { result.roots.forEach(checkConnected) }) }) + +describe("selectOverviewSpans errorsOnly", () => { + it("keeps only errors, their ancestors and the roots, even when the trace fits the budget", () => { + const roots = bigTree() + const result = selectOverviewSpans(roots, 100, { errorsOnly: true }) + const kept = ids(result.roots) + expect(kept.has("root")).toBe(true) + expect(kept.has("a")).toBe(true) + expect(kept.has("a2")).toBe(true) + expect(kept.has("a1")).toBe(false) + expect(kept.has("b")).toBe(false) + expect(kept.has("c")).toBe(false) + expect(result.renderedCount).toBe(3) + expect(result.truncated).toBe(true) + expect(result.omittedByParent.get("root")?.count).toBe(12) + }) +}) diff --git a/apps/api/src/mcp/lib/span-tree.ts b/apps/api/src/mcp/lib/span-tree.ts index c3aacffd2..c565a4e88 100644 --- a/apps/api/src/mcp/lib/span-tree.ts +++ b/apps/api/src/mcp/lib/span-tree.ts @@ -70,7 +70,19 @@ function subtreeSize(node: SpanNode): number { * long/structural spans. Returns the original tree unchanged when it already * fits within `budget` (so small traces render exactly as before). */ -export function selectOverviewSpans(roots: ReadonlyArray, budget: number): OverviewSelection { +export interface OverviewOptions { + /** + * Keep only error spans, their ancestor chains and the roots — no budget fill. Applies even + * to traces that would fit the budget, since the point is to strip the healthy spans. + */ + readonly errorsOnly?: boolean +} + +export function selectOverviewSpans( + roots: ReadonlyArray, + budget: number, + options: OverviewOptions = {}, +): OverviewSelection { let totalCount = 0 const parentOf = new Map() forEachNode(roots, (node, parent) => { @@ -78,7 +90,7 @@ export function selectOverviewSpans(roots: ReadonlyArray, budget: numb parentOf.set(key(node), parent) }) - if (totalCount <= budget) { + if (totalCount <= budget && !options.errorsOnly) { return { roots: roots as SpanNode[], renderedCount: totalCount, @@ -106,10 +118,12 @@ export function selectOverviewSpans(roots: ReadonlyArray, budget: numb }) // 2. Fill remaining budget by score, highest first. - candidates.sort((a, b) => score(b) - score(a)) - for (const node of candidates) { - if (selected.size >= budget) break - addWithAncestors(node) + if (!options.errorsOnly) { + candidates.sort((a, b) => score(b) - score(a)) + for (const node of candidates) { + if (selected.size >= budget) break + addWithAncestors(node) + } } // 3. Rebuild a pruned tree of new nodes; record omissions per parent. @@ -138,7 +152,7 @@ export function selectOverviewSpans(roots: ReadonlyArray, budget: numb roots: prunedRoots, renderedCount: selected.size, totalCount, - truncated: true, + truncated: selected.size < totalCount, omittedByParent, } } diff --git a/apps/api/src/mcp/tools/error-detail.ts b/apps/api/src/mcp/tools/error-detail.ts index 04ea340a2..b678fec02 100644 --- a/apps/api/src/mcp/tools/error-detail.ts +++ b/apps/api/src/mcp/tools/error-detail.ts @@ -112,7 +112,20 @@ export function registerErrorDetailTool(server: McpToolRegistrar) { ` Services: ${t.services.join(", ")}`, ` Time: ${t.startTime}`, ) - if (t.errorMessage) { + // The failing span first: name, service, status and the attributes that say what it + // was doing. That is the line an investigator needs; the logs below are context. + if (t.errorSpan) { + lines.push( + ` Error span: ${t.errorSpan.name} — ${t.errorSpan.serviceName} span=${t.errorSpan.spanId}`, + ) + if (t.errorSpan.statusMessage) { + lines.push(` Status: "${truncate(t.errorSpan.statusMessage, 160)}"`) + } + const attrs = Object.entries(t.errorSpan.attributes) + if (attrs.length > 0) { + lines.push(` {${attrs.map(([k, v]) => `${k}=${truncate(v, 60)}`).join(", ")}}`) + } + } else if (t.errorMessage) { lines.push(` Error: ${truncate(t.errorMessage, 120)}`) } if (t.logs.length > 0) { @@ -137,9 +150,10 @@ export function registerErrorDetailTool(server: McpToolRegistrar) { lines.push(``) } - const nextSteps = Arr.map( - Arr.take(result.traces, 3), - (t) => `\`inspect_trace trace_id="${t.traceId}"\` — full span tree`, + const nextSteps = Arr.map(Arr.take(result.traces, 3), (t) => + t.errorSpan + ? `\`inspect_span trace_id="${t.traceId}" span_id="${t.errorSpan.spanId}"\` — the failing span's full attributes` + : `\`inspect_trace trace_id="${t.traceId}" errors_only=true\` — the failing spans only`, ) nextSteps.push( `\`search_logs service="${service ?? ""}" severity="ERROR"\` — search for related error logs`, @@ -160,6 +174,9 @@ export function registerErrorDetailTool(server: McpToolRegistrar) { services: [...t.services], startTime: t.startTime, errorMessage: t.errorMessage || undefined, + errorSpan: t.errorSpan + ? { ...t.errorSpan, attributes: { ...t.errorSpan.attributes } } + : undefined, logs: Arr.map(t.logs, (l) => ({ ...l })), })), }, diff --git a/apps/api/src/mcp/tools/find-errors.ts b/apps/api/src/mcp/tools/find-errors.ts index 7cb0285ef..f9de413ab 100644 --- a/apps/api/src/mcp/tools/find-errors.ts +++ b/apps/api/src/mcp/tools/find-errors.ts @@ -1,8 +1,14 @@ -import { optionalNumberParam, optionalStringParam, optionalTimeParam, type McpToolRegistrar } from "./types" +import { + optionalNumberParam, + optionalStringParam, + optionalTimeParam, + validationError, + type McpToolRegistrar, +} from "./types" import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse" import { resolveTimeRange } from "@/mcp/lib/time" -import { formatNumber, formatTable } from "@/mcp/lib/format" +import { formatNumber, formatTable, truncate } from "@/mcp/lib/format" import { formatNextSteps } from "@/mcp/lib/next-steps" import { Array as Arr, Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" @@ -22,16 +28,40 @@ export function registerFindErrorsTool(server: McpToolRegistrar) { end_time: optionalTimeParam("End of time range (YYYY-MM-DD HH:mm:ss)"), service: optionalStringParam("Filter to a specific service"), environment: optionalStringParam("Filter by deployment environment (e.g. production, staging)"), + identity: optionalStringParam( + 'Pass "unexpected" to keep only identities that break a no-unknown-errors policy: labels outside `namespace_prefix` (library tags such as `AI.Error`, bare `Error`) plus the 5xx and unexpected-error-envelope markers. Omit for all errors.', + ), + namespace_prefix: optionalStringParam( + 'The prefix every deliberate, namespaced error tag starts with (default "@maple/"). Only used with identity="unexpected".', + ), limit: optionalNumberParam("Max results (default 20)"), }), - Effect.fn("McpTool.findErrors")(function* ({ start_time, end_time, service, environment, limit }) { + Effect.fn("McpTool.findErrors")(function* ({ + start_time, + end_time, + service, + environment, + identity, + namespace_prefix, + limit, + }) { + if (identity !== undefined && identity !== "unexpected") { + return validationError(`Invalid identity: '${identity}'. Pass "unexpected" or omit it.`) + } const { st, et } = resolveTimeRange(start_time, end_time) const tenant = yield* CurrentMcpTenant + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + service: service ?? "all", + identity: identity ?? "all", + }) const errors = yield* findErrors({ timeRange: { startTime: st, endTime: et }, service: service ?? undefined, environment: environment ?? undefined, + identity: identity ?? undefined, + namespacePrefix: namespace_prefix ?? undefined, limit: limit ?? 20, }).pipe( provideWarehouseExecutorFromTenant(tenant), @@ -39,14 +69,21 @@ export function registerFindErrorsTool(server: McpToolRegistrar) { ) if (errors.length === 0) { - return { content: [{ type: "text", text: `No errors found in ${st} — ${et}` }] } + const scope = identity === "unexpected" ? "unexpected-identity errors" : "errors" + return { content: [{ type: "text", text: `No ${scope} found in ${st} — ${et}` }] } } - const lines: string[] = [`## Errors by Type`, ``] + const lines: string[] = [ + identity === "unexpected" ? `## Unexpected Error Identities` : `## Errors by Type`, + ``, + ] - const headers = ["Error", "Fingerprint", "Count", "Affected Services", "Last Seen"] + // One occurrence's message per row: the same tag can own a dozen fingerprints, and the + // label alone gave no way to tell them apart short of an error_detail call each. + const headers = ["Error", "Message", "Fingerprint", "Count", "Affected Services", "Last Seen"] const rows = Arr.map(errors, (e) => [ - e.label.length > 60 ? e.label.slice(0, 57) + "..." : e.label, + truncate(e.label, 60), + truncate(e.sampleMessage.replace(/\s+/g, " ").replace(/\|/g, "\\|"), 80), e.fingerprintHash, formatNumber(e.count), String(e.affectedServicesCount), @@ -72,9 +109,11 @@ export function registerFindErrorsTool(server: McpToolRegistrar) { tool: "find_errors", data: { timeRange: { start: st, end: et }, + identity: identity ?? "all", errors: Arr.map(errors, (e) => ({ fingerprintHash: e.fingerprintHash, label: e.label, + sampleMessage: e.sampleMessage, count: e.count, affectedServicesCount: e.affectedServicesCount, lastSeen: e.lastSeen, diff --git a/apps/api/src/mcp/tools/inspect-trace.ts b/apps/api/src/mcp/tools/inspect-trace.ts index 5b931668d..5ce6b56e8 100644 --- a/apps/api/src/mcp/tools/inspect-trace.ts +++ b/apps/api/src/mcp/tools/inspect-trace.ts @@ -1,4 +1,11 @@ -import { requiredStringParam, optionalStringParam, type McpToolRegistrar } from "./types" +import { + optionalBooleanParam, + optionalNumberParam, + optionalStringParam, + requiredStringParam, + type McpToolRegistrar, +} from "./types" +import { clampLimit } from "@/mcp/lib/limits" import { warehouseToMcpHandlers } from "@/mcp/lib/map-warehouse-error" import { withTenantExecutor } from "@/mcp/lib/query-warehouse" import { formatNextSteps } from "@/mcp/lib/next-steps" @@ -14,6 +21,8 @@ import { inspectTrace, type SpanNode } from "@maple/query-engine/observability" * to this budget — deeper inspection goes through `inspect_span` / `search_traces`. */ const MAX_OVERVIEW_SPANS = 100 +/** Hard ceiling for `max_spans`; past this a single response stops being readable. */ +const MAX_OVERVIEW_SPANS_CEILING = 300 export function registerInspectTraceTool(server: McpToolRegistrar) { server.tool( @@ -24,9 +33,20 @@ export function registerInspectTraceTool(server: McpToolRegistrar) { timestamp: optionalStringParam( "ISO-8601 timestamp of any span in the trace (e.g. from `search_traces` results). Used to narrow the ClickHouse scan to a ±1h window — required for traces older than 24h, strongly recommended otherwise.", ), + errors_only: optionalBooleanParam( + "Render only error spans, their ancestors and the roots — the fastest way to read a large trace's failure without its healthy spans.", + ), + max_spans: optionalNumberParam( + `Spans to render before collapsing the rest (default ${MAX_OVERVIEW_SPANS}, max ${MAX_OVERVIEW_SPANS_CEILING}). Errors and roots are always kept.`, + ), }), - Effect.fn("McpTool.inspectTrace")(function* ({ trace_id, timestamp }) { + Effect.fn("McpTool.inspectTrace")(function* ({ trace_id, timestamp, errors_only, max_spans }) { yield* Effect.annotateCurrentSpan("traceId", trace_id) + const budget = clampLimit(max_spans, { + defaultValue: MAX_OVERVIEW_SPANS, + max: MAX_OVERVIEW_SPANS_CEILING, + }) + const options = { errorsOnly: errors_only === true } const timestampHint = timestamp ? new Date(timestamp) : undefined if (timestampHint && Number.isNaN(timestampHint.getTime())) { @@ -60,12 +80,14 @@ export function registerInspectTraceTool(server: McpToolRegistrar) { rootDurationMs: result.rootDurationMs, spans: result.spans, logs: result.logs, - budget: MAX_OVERVIEW_SPANS, + budget, + options, }) yield* Effect.annotateCurrentSpan({ "result.rowCount": result.spanCount, "result.renderedSpanCount": overview.renderedCount, + "result.errorsOnly": options.errorsOnly, }) const collectServices = (n: SpanNode): string[] => [ diff --git a/apps/api/src/mcp/tools/list-error-issues.ts b/apps/api/src/mcp/tools/list-error-issues.ts index bfe673fd6..50f283dfd 100644 --- a/apps/api/src/mcp/tools/list-error-issues.ts +++ b/apps/api/src/mcp/tools/list-error-issues.ts @@ -1,7 +1,9 @@ import { McpQueryError, + optionalBooleanParam, optionalNumberParam, optionalStringParam, + optionalTimeParam, validationError, type McpToolRegistrar, } from "./types" @@ -48,6 +50,12 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { "Filter by issue kind: error (fingerprint groups) or alert (alert-rule incidents)", ), service: optionalStringParam("Filter by service name"), + last_seen_after: optionalTimeParam( + 'Only issues that received an occurrence after this time (YYYY-MM-DD HH:mm:ss). The way to ask "what fired recently" without paging the whole backlog.', + ), + compact: optionalBooleanParam( + "Narrow table and payload: id, state, severity, service, exception, events, last seen, fingerprint. Omits assignment, lease and notes.", + ), limit: optionalNumberParam("Max results (default 50)"), include_archived: optionalStringParam("Pass '1' to include archived issues in results"), }), @@ -56,6 +64,8 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { severity, kind, service, + last_seen_after, + compact, limit, include_archived, }) { @@ -65,6 +75,8 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { workflowState: workflow_state ?? "all", severity: severity ?? "all", service: service ?? "all", + lastSeenAfter: last_seen_after ?? "none", + compact: compact === true, limit: limit ?? 50, }) const readModels = yield* ErrorIssueReadModelsService @@ -110,6 +122,7 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { severity: typedSeverity, kind: typedKind, service, + startTime: last_seen_after ?? undefined, limit: limit ?? 50, includeArchived: include_archived === "1", }) @@ -132,6 +145,28 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { if (issues.length === 0) { lines.push("No error issues found.") + } else if (compact) { + const headers = [ + "Issue ID", + "State", + "Severity", + "Service", + "Exception", + "Events", + "Last seen", + "Fingerprint", + ] + const rows = issues.map((i) => [ + i.id, + i.hasOpenIncident ? `${i.workflowState} (incident)` : i.workflowState, + i.severity ?? "—", + i.serviceName, + truncate(i.errorLabel || `${i.exceptionType}: ${i.exceptionMessage}`, 50), + formatNumber(i.occurrenceCount), + i.lastSeenAt.slice(0, 19), + i.fingerprintHash, + ]) + lines.push(formatTable(headers, rows)) } else { const headers = [ // Full id, not a prefix. The 8-char truncation this used to render was @@ -152,6 +187,9 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { "Last seen", "Assigned", "Holder", + // The warehouse identity, so an issue can go straight to error_detail + // instead of being re-derived through find_errors. + "Fingerprint", ] const rows = issues.map((i) => [ i.id, @@ -174,6 +212,7 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { ? `agent:${i.leaseHolder.agentName ?? "?"}` : (i.leaseHolder.userId ?? "user") : "—", + i.fingerprintHash, ]) lines.push(formatTable(headers, rows)) } @@ -191,6 +230,12 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { } and regressed; read what was already tried before investigating it as new`, ) } + const topError = issues.find((i) => i.kind === "error") + if (topError) { + nextSteps.push( + `\`error_detail fingerprint="${topError.fingerprintHash}"\` — sample traces for the most recent issue`, + ) + } for (const id of triageIds) { nextSteps.push(`\`claim_error_issue issue_id="${id}"\` — pick up this issue`) nextSteps.push( @@ -203,46 +248,63 @@ export function registerListErrorIssuesTool(server: McpToolRegistrar) { content: createDualContent(lines.join("\n"), { tool: "list_error_issues", data: { - issues: issues.map((i) => ({ - id: i.id, - kind: i.kind, - fingerprintHash: i.fingerprintHash, - workflowState: i.workflowState, - priority: i.priority, - severity: i.severity, - severitySource: i.severitySource, - serviceName: i.serviceName, - errorLabel: i.errorLabel, - exceptionType: i.exceptionType, - exceptionMessage: i.exceptionMessage, - topFrame: i.topFrame, - occurrenceCount: i.occurrenceCount, - firstSeenAt: i.firstSeenAt, - lastSeenAt: i.lastSeenAt, - assignedActor: i.assignedActor - ? { - id: i.assignedActor.id, - type: i.assignedActor.type, - userId: i.assignedActor.userId, - agentName: i.assignedActor.agentName, - model: i.assignedActor.model, - capabilities: i.assignedActor.capabilities, - } - : null, - leaseHolder: i.leaseHolder - ? { - id: i.leaseHolder.id, - type: i.leaseHolder.type, - userId: i.leaseHolder.userId, - agentName: i.leaseHolder.agentName, - model: i.leaseHolder.model, - capabilities: i.leaseHolder.capabilities, - } - : null, - leaseExpiresAt: i.leaseExpiresAt, - notes: i.notes, - hasOpenIncident: i.hasOpenIncident, - })), + compact: compact === true, + issues: compact + ? issues.map((i) => ({ + id: i.id, + kind: i.kind, + fingerprintHash: i.fingerprintHash, + workflowState: i.workflowState, + severity: i.severity, + serviceName: i.serviceName, + errorLabel: i.errorLabel, + occurrenceCount: i.occurrenceCount, + firstSeenAt: i.firstSeenAt, + lastSeenAt: i.lastSeenAt, + regressionCount: i.regressionCount, + lastResolvedAt: i.lastResolvedAt, + hasOpenIncident: i.hasOpenIncident, + })) + : issues.map((i) => ({ + id: i.id, + kind: i.kind, + fingerprintHash: i.fingerprintHash, + workflowState: i.workflowState, + priority: i.priority, + severity: i.severity, + severitySource: i.severitySource, + serviceName: i.serviceName, + errorLabel: i.errorLabel, + exceptionType: i.exceptionType, + exceptionMessage: i.exceptionMessage, + topFrame: i.topFrame, + occurrenceCount: i.occurrenceCount, + firstSeenAt: i.firstSeenAt, + lastSeenAt: i.lastSeenAt, + assignedActor: i.assignedActor + ? { + id: i.assignedActor.id, + type: i.assignedActor.type, + userId: i.assignedActor.userId, + agentName: i.assignedActor.agentName, + model: i.assignedActor.model, + capabilities: i.assignedActor.capabilities, + } + : null, + leaseHolder: i.leaseHolder + ? { + id: i.leaseHolder.id, + type: i.leaseHolder.type, + userId: i.leaseHolder.userId, + agentName: i.leaseHolder.agentName, + model: i.leaseHolder.model, + capabilities: i.leaseHolder.capabilities, + } + : null, + leaseExpiresAt: i.leaseExpiresAt, + notes: i.notes, + hasOpenIncident: i.hasOpenIncident, + })), total: issues.length, }, }), diff --git a/apps/api/src/services/warehouse/warehouse-catalog.ts b/apps/api/src/services/warehouse/warehouse-catalog.ts index b8e0d7022..f97dd071c 100644 --- a/apps/api/src/services/warehouse/warehouse-catalog.ts +++ b/apps/api/src/services/warehouse/warehouse-catalog.ts @@ -14,7 +14,7 @@ import * as Datasources from "@maple/domain/tinybird" const TABLE_NOTES: Record> = { logs: [ - "`SeverityText` values are Title Case: 'Trace', 'Debug', 'Info', 'Warn', 'Error', 'Fatal'. Filter with `SeverityText = 'Error'` (NOT `'ERROR'`).", + "`SeverityText` casing varies by SDK ('Error' from Effect services, 'ERROR' from OTel SDKs), so filter on the OTel level number instead: `SeverityNumber BETWEEN 17 AND 20` is ERROR, 13-16 WARN, 9-12 INFO, 5-8 DEBUG, 1-4 TRACE, 21-24 FATAL. Where you must use text, use `upper(SeverityText) = 'ERROR'`.", "`SeverityNumber` follows OTel: 1-4 Trace, 5-8 Debug, 9-12 Info, 13-16 Warn, 17-20 Error, 21-24 Fatal.", "`ResourceAttributes` and `LogAttributes` are `Map(LowCardinality(String), String)` — access with `LogAttributes['key']`; missing keys return '' (empty string), not NULL.", "Use `TimestampTime` (DateTime) for `$__timeFilter(TimestampTime)` if you want sort-key-prefix-friendly filtering; `Timestamp` is DateTime64 (nanosecond precision).", diff --git a/apps/cli/src/core/remote-ops.ts b/apps/cli/src/core/remote-ops.ts index bfda857e8..4635717e6 100644 --- a/apps/cli/src/core/remote-ops.ts +++ b/apps/cli/src/core/remote-ops.ts @@ -522,6 +522,25 @@ export const tracesBreakdown = ( * span count, participating services, the root span's name. It is bounded by * the caller's own `--limit`. */ +/** The same attribute allowlist the warehouse query projects for the failing span. */ +const ERROR_SPAN_ATTRIBUTE_KEYS = [ + "gen_ai.request.model", + "gen_ai.tool.name", + "http.request.method", + "http.route", + "query.context", + "error.type", +] as const + +const pickErrorSpanAttributes = (attributes: Record): Record => { + const picked: Record = {} + for (const key of ERROR_SPAN_ATTRIBUTE_KEYS) { + const value = attributes[key] + if (typeof value === "string" && value !== "") picked[key] = value + } + return picked +} + export const errorDetail = ( client: MapleV2Client, p: { fingerprintHash: string; range: Range; service?: string; limit?: number }, @@ -571,6 +590,7 @@ export const errorDetail = ( }) : undefined const root = trace.spans.find((s) => s.parent_span_id === null) + const failing = trace.spans.find((s) => s.status_code === "Error") return { traceId: sample.trace_id, rootSpanName: root?.name ?? "", @@ -579,6 +599,15 @@ export const errorDetail = ( services: Array.from(new Set(trace.spans.map((s) => s.service_name))), startTime: sample.timestamp, errorMessage: sample.exception_message, + errorSpan: failing + ? { + spanId: failing.id, + name: failing.name, + serviceName: failing.service_name, + statusMessage: failing.status_message ?? "", + attributes: pickErrorSpanAttributes(failing.attributes), + } + : undefined, logs: (logs?.data ?? []).slice(0, 5).map((l) => ({ timestamp: l.timestamp, severityText: l.severity_text || "INFO", diff --git a/knip.json b/knip.json index 0b5588007..f228106d4 100644 --- a/knip.json +++ b/knip.json @@ -25,7 +25,15 @@ "apps/api": { // src/worker.ts was auto-detected from wrangler.jsonc's `main` until the // wrangler configs were deleted; the Worker entries are now named here. - "entry": ["src/worker.ts", "alchemy.run.ts", "autumn.config.ts", "scripts/cold-path/*.mjs"], + // bench-suites/*.ts are loaded by path from scripts/bench-queries.ts (a dynamic + // import off the CLI's --suite argument), so knip cannot see the edge. + "entry": [ + "src/worker.ts", + "alchemy.run.ts", + "autumn.config.ts", + "scripts/cold-path/*.mjs", + "scripts/bench-suites/*.ts" + ], // Shelled out to by scripts/bench-startup-cpu.ts, never imported. "ignoreDependencies": ["cloudflare"] }, diff --git a/packages/domain/src/mcp-structured-types.ts b/packages/domain/src/mcp-structured-types.ts index 23dff17bb..dd07ff304 100644 --- a/packages/domain/src/mcp-structured-types.ts +++ b/packages/domain/src/mcp-structured-types.ts @@ -72,6 +72,8 @@ export interface FindSlowTracesData { export interface ErrorTypeRow { fingerprintHash: string label: string + /** One occurrence's status message, to tell fingerprints with the same label apart. */ + sampleMessage: string count: number affectedServicesCount: number lastSeen: string @@ -79,9 +81,20 @@ export interface ErrorTypeRow { export interface FindErrorsData { timeRange: { start: string; end: string } + /** "all", or "unexpected" when the list was narrowed to policy-violating identities. */ + identity: string errors: ErrorTypeRow[] } +/** The span that failed inside a sampled trace, with the attributes that say what it was doing. */ +export interface ErrorDetailSpanSummary { + spanId: string + name: string + serviceName: string + statusMessage: string + attributes: Record +} + export interface ErrorDetailTrace { traceId: string rootSpanName: string @@ -90,6 +103,7 @@ export interface ErrorDetailTrace { services: string[] startTime: string errorMessage?: string + errorSpan?: ErrorDetailSpanSummary logs: Array<{ timestamp: string severityText: string @@ -661,8 +675,26 @@ export interface ErrorIssueRow { hasOpenIncident: boolean } +/** The `compact: true` row — identity, state and volume; no assignment, lease or notes. */ +export interface ErrorIssueCompactRow { + id: string + kind: string + fingerprintHash: string + workflowState: string + severity: string | null + serviceName: string + errorLabel: string + occurrenceCount: number + firstSeenAt: string + lastSeenAt: string + regressionCount: number + lastResolvedAt: string | null + hasOpenIncident: boolean +} + export interface ListErrorIssuesData { - issues: ErrorIssueRow[] + compact: boolean + issues: ErrorIssueRow[] | ErrorIssueCompactRow[] total: number } diff --git a/packages/domain/src/tinybird/endpoints.ts b/packages/domain/src/tinybird/endpoints.ts index 619fb0adf..55adec476 100644 --- a/packages/domain/src/tinybird/endpoints.ts +++ b/packages/domain/src/tinybird/endpoints.ts @@ -418,6 +418,9 @@ export interface ErrorsByTypeParams { limit?: number exclude_spam_patterns?: string root_only?: boolean + /** "unexpected" keeps only identities outside `namespace_prefix` plus the 5xx/unexpected-envelope markers. */ + identity?: string + namespace_prefix?: string } // errors_timeseries @@ -447,6 +450,15 @@ export interface ErrorDetailTracesOutput { readonly services: readonly string[] readonly rootSpanName: string readonly errorMessage: string + readonly errorSpanId: string + readonly errorSpanName: string + readonly errorServiceName: string + readonly errorModel: string + readonly errorToolName: string + readonly errorHttpMethod: string + readonly errorHttpRoute: string + readonly errorQueryContext: string + readonly errorType: string } export interface ErrorDetailTracesParams { diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 1126bfa5c..b9731a98a 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -5557,7 +5557,7 @@ SELECT ORDER BY bucket ASC, groupName ASC FORMAT JSON --- pipe:error_detail_traces:default:baseline [99fb32db] +-- pipe:error_detail_traces:default:baseline [ce04a4f1] SELECT TraceId AS traceId, min(Timestamp) AS startTime, @@ -5565,7 +5565,16 @@ SELECT count() AS spanCount, groupUniqArray(ServiceName) AS services, anyIf(SpanName, ParentSpanId = '') AS rootSpanName, - any(StatusMessage) AS errorMessage + anyIf(StatusMessage, StatusCode = 'Error') AS errorMessage, + anyIf(SpanId, StatusCode = 'Error') AS errorSpanId, + anyIf(SpanName, StatusCode = 'Error') AS errorSpanName, + anyIf(ServiceName, StatusCode = 'Error') AS errorServiceName, + anyIf(SpanAttributes['gen_ai.request.model'], StatusCode = 'Error') AS errorModel, + anyIf(SpanAttributes['gen_ai.tool.name'], StatusCode = 'Error') AS errorToolName, + anyIf(SpanAttributes['http.request.method'], StatusCode = 'Error') AS errorHttpMethod, + anyIf(SpanAttributes['http.route'], StatusCode = 'Error') AS errorHttpRoute, + anyIf(SpanAttributes['query.context'], StatusCode = 'Error') AS errorQueryContext, + anyIf(SpanAttributes['error.type'], StatusCode = 'Error') AS errorType FROM trace_detail_spans WHERE OrgId = 'org_sql_catalog' AND TraceId IN (SELECT @@ -5727,6 +5736,25 @@ SELECT LIMIT 1 FORMAT JSON +-- pipe:errors_by_type:unexpected-identity:baseline [1457ff9b] +SELECT + toString(FingerprintHash) AS fingerprintHash, + any(ErrorLabel) AS errorLabel, + any(StatusMessage) AS sampleMessage, + count() AS count, + uniq(ServiceName) AS affectedServicesCount, + min(Timestamp) AS firstSeen, + max(Timestamp) AS lastSeen + FROM error_events_by_time + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND (ErrorLabel NOT LIKE '@maple/%' OR ErrorLabel IN ('@maple/api/http/Http5xxResponseError', '@maple/http/v2/UnexpectedError', '@maple/http/v1/V1UnexpectedError')) + GROUP BY fingerprintHash + ORDER BY count DESC + LIMIT 50 + FORMAT JSON + -- pipe:errors_facets:default:baseline [fe66f114] SELECT ServiceName AS name, @@ -6065,7 +6093,7 @@ SELECT LIMIT 50 FORMAT JSON --- pipe:list_logs:searched:baseline [f4e5cc5d] +-- pipe:list_logs:searched:baseline [af6a1e45] SELECT Timestamp AS timestamp, SeverityText AS severityText, @@ -6084,7 +6112,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND Body ILIKE '%connection refused%' AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -6096,7 +6124,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND Body ILIKE '%connection refused%' ORDER BY ts DESC @@ -6105,7 +6133,7 @@ SELECT LIMIT 50 FORMAT JSON --- pipe:list_logs:searched:bloom [efc50db5] +-- pipe:list_logs:searched:bloom [d1878c6d] SELECT Timestamp AS timestamp, SeverityText AS severityText, @@ -6124,7 +6152,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND ((hasToken(lower(Body), 'connection') AND hasToken(lower(Body), 'refused')) AND Body ILIKE '%connection refused%') AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -6136,7 +6164,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND ((hasToken(lower(Body), 'connection') AND hasToken(lower(Body), 'refused')) AND Body ILIKE '%connection refused%') ORDER BY ts DESC @@ -6145,7 +6173,7 @@ SELECT LIMIT 50 FORMAT JSON --- pipe:list_logs:searched:text [fad2e4f1] +-- pipe:list_logs:searched:text [7eef92d1] SELECT Timestamp AS timestamp, SeverityText AS severityText, @@ -6164,7 +6192,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND (hasAllTokens(lower(Body), 'connection refused') AND Body ILIKE '%connection refused%') AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -6176,7 +6204,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND SeverityText = 'ERROR' + AND SeverityText IN ('ERROR', 'Error', 'error') AND TraceId = '0af7651916cd43dd8448eb211c80319c' AND (hasAllTokens(lower(Body), 'connection refused') AND Body ILIKE '%connection refused%') ORDER BY ts DESC diff --git a/packages/query-engine/src/benchmark/catalog.ts b/packages/query-engine/src/benchmark/catalog.ts index e2fe1f199..358d05e69 100644 --- a/packages/query-engine/src/benchmark/catalog.ts +++ b/packages/query-engine/src/benchmark/catalog.ts @@ -327,6 +327,7 @@ export const pipeFixtures: ReadonlyArray = [ label: "fingerprint-scoped", params: { fingerprint_hashes: FINGERPRINT, deployment_envs: "production", limit: 1 }, }, + { pipe: "errors_by_type", label: "unexpected-identity", params: { identity: "unexpected" } }, { pipe: "errors_timeseries", label: "default", params: { fingerprint_hash: FINGERPRINT } }, { pipe: "errors_facets", label: "default", params: {} }, { pipe: "errors_summary", label: "default", params: {} }, diff --git a/packages/query-engine/src/ch/pipe-dispatch.ts b/packages/query-engine/src/ch/pipe-dispatch.ts index 0f167979f..5e4f9d24a 100644 --- a/packages/query-engine/src/ch/pipe-dispatch.ts +++ b/packages/query-engine/src/ch/pipe-dispatch.ts @@ -13,6 +13,7 @@ // so the two adapters are not duplicates. import type { TracesMetric, AttributeFilter, MetricType } from "@maple/domain/query-engine" +import { DEFAULT_ERROR_NAMESPACE_PREFIX, UNEXPECTED_IDENTITY_MARKERS } from "./queries/errors" import type { OrgId } from "@maple/domain" import { compile, compileUnion, type CompiledQuery } from "@maple-dev/clickhouse-builder" import { rawCompiledQuery } from "./raw-sql" @@ -475,6 +476,14 @@ export function compilePipeQuery( services: str("services")?.split(",").filter(Boolean), deploymentEnvs: str("deployment_envs")?.split(",").filter(Boolean), fingerprintHashes: str("fingerprint_hashes")?.split(",").filter(Boolean), + unexpectedIdentity: + str("identity") === "unexpected" + ? { + namespacePrefix: + str("namespace_prefix") ?? DEFAULT_ERROR_NAMESPACE_PREFIX, + markerLabels: UNEXPECTED_IDENTITY_MARKERS, + } + : undefined, limit: int("limit", 50), }), { orgId, startTime, endTime }, diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts index 7c2c9a7e3..0e3a5d14b 100644 --- a/packages/query-engine/src/ch/queries/errors.test.ts +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -115,6 +115,25 @@ describe("errorsByTypeQuery", () => { const { sql } = compileUnsafe(q, baseParams) expect(sql).toContain("LIMIT 25") }) + + it("keeps only unexpected identities: outside the namespace, or a 5xx/envelope marker", () => { + const q = errorsByTypeQuery({ + unexpectedIdentity: { + namespacePrefix: "@maple/", + markerLabels: ["@maple/api/http/Http5xxResponseError"], + }, + }) + const { sql } = compileUnsafe(q, baseParams) + expect(sql).toContain("ErrorLabel NOT LIKE '@maple/%'") + expect(sql).toContain("ErrorLabel IN ('@maple/api/http/Http5xxResponseError')") + expect(sql).toContain("any(StatusMessage) AS sampleMessage") + }) + + it("escapes LIKE wildcards in the namespace prefix", () => { + const q = errorsByTypeQuery({ unexpectedIdentity: { namespacePrefix: "my_app%", markerLabels: [] } }) + const { sql } = compileUnsafe(q, baseParams) + expect(sql).toContain("NOT LIKE 'my\\\\_app\\\\%%'") + }) }) // errorsTimeseriesQuery @@ -217,6 +236,14 @@ describe("errorDetailTracesQuery", () => { // The limit applies to the error subquery expect(sql).toContain("LIMIT 20") }) + it("reports the failing span rather than an arbitrary one", () => { + const q = errorDetailTracesQuery({ fingerprintHash: "1" }) + const { sql } = compileUnsafe(q, baseParams) + expect(sql).toContain("anyIf(StatusMessage, StatusCode = 'Error') AS errorMessage") + expect(sql).toContain("anyIf(SpanId, StatusCode = 'Error') AS errorSpanId") + expect(sql).toContain("anyIf(SpanName, StatusCode = 'Error') AS errorSpanName") + expect(sql).toContain("SpanAttributes['gen_ai.request.model']") + }) }) // Exclusions on the fingerprint-resolving query. The errors list is issue-first until a facet is diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 0e6c05242..55f3fd177 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -142,9 +142,28 @@ const sharedFilterConditions = ( // hash (string), not a query-time heuristic — see materializations.ts / // fingerprint.ts for how the hash + label are derived. +/** + * Error identities that violate the "every failure is a namespaced tagged error" policy: labels + * outside the org's own namespace (library tags such as `AI.Error`, bare `Error`), plus the + * markers Maple emits when a request ended in a 5xx or the unexpected-error envelope. + */ +export interface UnexpectedIdentityFilter { + readonly namespacePrefix: string + readonly markerLabels: readonly string[] +} + +export const DEFAULT_ERROR_NAMESPACE_PREFIX = "@maple/" + +export const UNEXPECTED_IDENTITY_MARKERS: readonly string[] = [ + "@maple/api/http/Http5xxResponseError", + "@maple/http/v2/UnexpectedError", + "@maple/http/v1/V1UnexpectedError", +] + export interface ErrorsByTypeOpts extends ErrorsSharedFilters { rootOnly?: boolean fingerprintHashes?: readonly string[] + unexpectedIdentity?: UnexpectedIdentityFilter limit?: number } @@ -178,6 +197,11 @@ export function errorsByTypeQuery(opts: ErrorsByTypeOpts) { opts.fingerprintHashes?.length ? fingerprintHashIn($.FingerprintHash, opts.fingerprintHashes) : undefined, + opts.unexpectedIdentity + ? $.ErrorLabel.notLike(`${likeLiteral(opts.unexpectedIdentity.namespacePrefix)}%`).or( + CH.inList($.ErrorLabel, opts.unexpectedIdentity.markerLabels), + ) + : undefined, ]) .groupBy("fingerprintHash") .orderBy(["count", "desc"]) @@ -185,6 +209,9 @@ export function errorsByTypeQuery(opts: ErrorsByTypeOpts) { .format("JSON") } +/** A namespace prefix is a literal, so its `%`/`_` must not act as LIKE wildcards. */ +const likeLiteral = (value: string): string => value.replace(/[\\%_]/g, (c) => `\\${c}`) + // Errors timeseries export interface ErrorsTimeseriesOpts { @@ -1242,6 +1269,15 @@ export interface ErrorDetailTracesOutput { readonly services: readonly string[] readonly rootSpanName: string readonly errorMessage: string + readonly errorSpanId: string + readonly errorSpanName: string + readonly errorServiceName: string + readonly errorModel: string + readonly errorToolName: string + readonly errorHttpMethod: string + readonly errorHttpRoute: string + readonly errorQueryContext: string + readonly errorType: string } export function errorDetailTracesQuery(opts: ErrorDetailTracesOpts) { @@ -1280,7 +1316,18 @@ export function errorDetailTracesQuery(opts: ErrorDetailTracesOpts) { spanCount: CH.count(), services: CH.groupUniqArray($.ServiceName), rootSpanName: CH.anyIf($.SpanName, $.ParentSpanId.eq("")), - errorMessage: CH.any_($.StatusMessage), + // The failing span, not an arbitrary one: `any(StatusMessage)` used to pick whichever span + // ClickHouse read first, which for most traces is a healthy span with an empty message. + errorMessage: CH.anyIf($.StatusMessage, $.StatusCode.eq("Error")), + errorSpanId: CH.anyIf($.SpanId, $.StatusCode.eq("Error")), + errorSpanName: CH.anyIf($.SpanName, $.StatusCode.eq("Error")), + errorServiceName: CH.anyIf($.ServiceName, $.StatusCode.eq("Error")), + errorModel: CH.anyIf($.SpanAttributes.get("gen_ai.request.model"), $.StatusCode.eq("Error")), + errorToolName: CH.anyIf($.SpanAttributes.get("gen_ai.tool.name"), $.StatusCode.eq("Error")), + errorHttpMethod: CH.anyIf($.SpanAttributes.get("http.request.method"), $.StatusCode.eq("Error")), + errorHttpRoute: CH.anyIf($.SpanAttributes.get("http.route"), $.StatusCode.eq("Error")), + errorQueryContext: CH.anyIf($.SpanAttributes.get("query.context"), $.StatusCode.eq("Error")), + errorType: CH.anyIf($.SpanAttributes.get("error.type"), $.StatusCode.eq("Error")), })) .where(($) => [ $.OrgId.eq(param.string("orgId")), diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts index 77d9a886c..99d653eee 100644 --- a/packages/query-engine/src/ch/queries/logs.test.ts +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -117,7 +117,14 @@ describe("logsTimeseriesQuery", () => { it("applies severity filter", () => { const q = logsTimeseriesQuery({ severity: "ERROR" }) const { sql } = compileUnsafe(q, baseParams) - expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") + }) + + it("matches a severity level across SDK spellings, but exact facet values as given", () => { + const level = logsTimeseriesQuery({ severity: "error" }) + expect(compileUnsafe(level, baseParams).sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") + const exact = logsTimeseriesQuery({ severities: ["Error"] }) + expect(compileUnsafe(exact, baseParams).sql).toContain("SeverityText = 'Error'") }) it("uses text-index candidates for multi-token body search and retains exact semantics", () => { @@ -323,7 +330,7 @@ describe("logsBreakdownQuery", () => { const q = logsBreakdownQuery({ groupBy: "service", serviceName: "api", severity: "ERROR" }) const { sql } = compileUnsafe(q, baseParams) expect(sql).toContain("ServiceName = 'api'") - expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") }) it("falls back to raw logs for contains-mode environment match", () => { @@ -432,7 +439,7 @@ describe("logsCountQuery", () => { expect(sql).toContain("FROM logs") expect(sql).not.toContain("logs_aggregates_hourly") expect(sql).toContain("ServiceName = 'api'") - expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") expect(sql).toContain("Body ILIKE '%timeout%'") }) }) @@ -507,7 +514,7 @@ describe("logsListQuery", () => { }) const { sql } = compileUnsafe(q, baseParams) expect(sql).toContain("ServiceName = 'api'") - expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") expect(sql).toContain("TraceId = 'trace123'") expect(sql).toContain("SpanId = 'span456'") expect(sql).toContain("Body ILIKE '%timeout%'") @@ -546,7 +553,7 @@ describe("logsListQuery", () => { const { sql } = compileUnsafe(q, baseParams) // Each filter appears twice — once per stage. expect(sql.match(/ServiceName = 'api'/g)).toHaveLength(2) - expect(sql.match(/SeverityText = 'ERROR'/g)).toHaveLength(2) + expect(sql.match(/SeverityText IN \('ERROR', 'Error', 'error'\)/g)).toHaveLength(2) expect(sql.match(/OrgId = 'org_1'/g)).toHaveLength(2) }) }) @@ -655,7 +662,7 @@ describe("logsFacetsQuery", () => { const q = logsFacetsQuery({ serviceName: "api", severity: "ERROR" }) const { sql } = compileUnionUnsafe(q, baseParams) expect(sql).toContain("ServiceName = 'api'") - expect(sql).toContain("SeverityText = 'ERROR'") + expect(sql).toContain("SeverityText IN ('ERROR', 'Error', 'error')") }) it("falls back to raw `logs` for `contains`-mode environment match", () => { diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 3a416e2a7..ea6cf80ab 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -16,7 +16,7 @@ import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition } from "../../traces-shared" import type { AttributeIndexMode, LogBodySearchMode } from "../../capabilities" import { edgeCondition, interiorConditions } from "./rollup-splice" -import { inclusionCondition, inclusionValues, soleValue } from "./query-helpers" +import { inclusionCondition, inclusionValues, severitySpellings, soleValue } from "./query-helpers" // Shared options @@ -151,7 +151,13 @@ function serviceSeverityConditions( opts: LogsQueryOpts, ): Array { const services = inclusionValues(opts.serviceName, opts.serviceNames) - const severities = inclusionValues(opts.severity, opts.severities) + // The scalar is a level ("ERROR") and matches every spelling; the array holds exact facet + // values the caller read back from the data, so it stays exact. + const severities = opts.severities?.length + ? opts.severities + : opts.severity + ? severitySpellings(opts.severity) + : undefined return [ services ? inclusionCondition($.ServiceName, services) : undefined, severities ? inclusionCondition($.SeverityText, severities) : undefined, @@ -790,7 +796,7 @@ function logsFacetsQueryFromMv( $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - CH.when(opts.severity, (v: string) => $.SeverityText.eq(v)), + CH.when(opts.severity, (v: string) => inclusionCondition($.SeverityText, severitySpellings(v))), opts.environments?.length ? CH.inList($.DeploymentEnv, opts.environments) : undefined, mvNamespaceCondition($, opts), ] @@ -867,7 +873,7 @@ function logsFacetsQueryFromRaw( $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), - CH.when(opts.severity, (v: string) => $.SeverityText.eq(v)), + CH.when(opts.severity, (v: string) => inclusionCondition($.SeverityText, severitySpellings(v))), environmentCondition($, opts), namespaceCondition($, opts), ] diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index b3aaf1f42..7a2c8afa8 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -190,6 +190,21 @@ export const facetAttrExpr = ( export const soleValue = (values: readonly A[]): A | undefined => values.length === 1 ? values[0] : undefined +/** + * Every spelling a severity *level* reaches the warehouse as. Effect's logger writes Title Case + * (`Error`), the OTel SDKs upper-case (`ERROR`), pino-style shims lower-case — so `severity: "ERROR"` + * matched none of Maple's own services. Exact values (kept as `IN`) preserve the sorting-key prefix + * on `logs_aggregates_hourly`, which `upper(SeverityText)` would not. + */ +export function severitySpellings(level: string): readonly string[] { + const trimmed = level.trim() + if (trimmed === "") return [] + const upper = trimmed.toUpperCase() + const lower = trimmed.toLowerCase() + const title = upper.charAt(0) + lower.slice(1) + return [...new Set([upper, title, lower])] +} + export function inclusionCondition(col: CH.Expr, values: readonly string[]): CH.Condition { const only = soleValue(values) return only === undefined ? CH.inList(col, values) : col.eq(only) diff --git a/packages/query-engine/src/observability/error-detail.test.ts b/packages/query-engine/src/observability/error-detail.test.ts index 3385866eb..7a26a465b 100644 --- a/packages/query-engine/src/observability/error-detail.test.ts +++ b/packages/query-engine/src/observability/error-detail.test.ts @@ -17,6 +17,15 @@ const traceRow = (traceId: string, startTime: string) => ({ services: ["api"], rootSpanName: "GET /", errorMessage: "boom", + errorSpanId: "span-err", + errorSpanName: "chat gpt-x", + errorServiceName: "api", + errorModel: "gpt-x", + errorToolName: "", + errorHttpMethod: "", + errorHttpRoute: "", + errorQueryContext: "", + errorType: "", }) const makeMockExecutor = ( @@ -76,4 +85,20 @@ describe("errorDetail", () => { assert.strictEqual(logs[0]!.params.end_time, timeRange.endTime) }), ) + + it.effect("surfaces the failing span with only the attributes it carries", () => + Effect.gen(function* () { + const captured: CapturedCalls = { pipeCalls: [] } + const result = yield* errorDetail({ fingerprintHash: "123", timeRange }).pipe( + Effect.provide( + makeLayer(makeMockExecutor(captured, [traceRow("t1", "2026-04-03 12:00:00")])), + ), + ) + const span = result.traces[0]!.errorSpan + assert.isDefined(span) + assert.strictEqual(span!.name, "chat gpt-x") + assert.strictEqual(span!.statusMessage, "boom") + assert.deepStrictEqual(span!.attributes, { "gen_ai.request.model": "gpt-x" }) + }), + ) }) diff --git a/packages/query-engine/src/observability/error-detail.ts b/packages/query-engine/src/observability/error-detail.ts index a304c022f..05a6063ee 100644 --- a/packages/query-engine/src/observability/error-detail.ts +++ b/packages/query-engine/src/observability/error-detail.ts @@ -21,6 +21,15 @@ const logRangeAround = (traceStartTime: string): { start_time: string; end_time: } } +/** The span that failed, with the handful of attributes that identify what it was doing. */ +export interface ErrorDetailSpan { + readonly spanId: string + readonly name: string + readonly serviceName: string + readonly statusMessage: string + readonly attributes: Readonly> +} + export interface ErrorDetailTrace { readonly traceId: string readonly rootSpanName: string @@ -29,9 +38,31 @@ export interface ErrorDetailTrace { readonly services: readonly string[] readonly startTime: string readonly errorMessage: string + readonly errorSpan: ErrorDetailSpan | undefined readonly logs: ReadonlyArray<{ timestamp: string; severityText: string; body: string }> } +const errorSpanOf = (t: ErrorDetailTracesOutput): ErrorDetailSpan | undefined => { + if (!t.errorSpanId) return undefined + const attributes: Record = {} + const attr = (key: string, value: string | undefined) => { + if (value) attributes[key] = value + } + attr("gen_ai.request.model", t.errorModel) + attr("gen_ai.tool.name", t.errorToolName) + attr("http.request.method", t.errorHttpMethod) + attr("http.route", t.errorHttpRoute) + attr("query.context", t.errorQueryContext) + attr("error.type", t.errorType) + return { + spanId: t.errorSpanId, + name: t.errorSpanName ?? "", + serviceName: t.errorServiceName ?? "", + statusMessage: t.errorMessage ?? "", + attributes, + } +} + export interface ErrorDetailOutput { readonly fingerprintHash: string readonly timeRange: TimeRange @@ -129,6 +160,7 @@ export const errorDetail = Effect.fn("Observability.errorDetail")(function* (inp services: t.services, startTime: t.startTime, errorMessage: t.errorMessage ?? "", + errorSpan: errorSpanOf(t), logs: pipe( logsResults[i]?.data ?? [], Arr.take(5), diff --git a/packages/query-engine/src/observability/find-errors.ts b/packages/query-engine/src/observability/find-errors.ts index 3ce8949f2..e25b561be 100644 --- a/packages/query-engine/src/observability/find-errors.ts +++ b/packages/query-engine/src/observability/find-errors.ts @@ -14,6 +14,8 @@ export const findErrors = Effect.fn("Observability.findErrors")(function* (input end_time: input.timeRange.endTime, ...(input.service && { services: input.service }), ...(input.environment && { deployment_envs: input.environment }), + ...(input.identity && { identity: input.identity }), + ...(input.namespacePrefix && { namespace_prefix: input.namespacePrefix }), limit: input.limit ?? 20, }, { profile: "aggregation" }, diff --git a/packages/query-engine/src/observability/index.ts b/packages/query-engine/src/observability/index.ts index 235b8c8da..3f976caad 100644 --- a/packages/query-engine/src/observability/index.ts +++ b/packages/query-engine/src/observability/index.ts @@ -23,7 +23,12 @@ export { searchTraces } from "./search-traces" export { inspectTrace } from "./inspect-trace" export { spanDetail, type SpanDetailInput, type SpanDetailResult } from "./span-detail" export { findErrors } from "./find-errors" -export { errorDetail, type ErrorDetailTrace, type ErrorDetailOutput } from "./error-detail" +export { + errorDetail, + type ErrorDetailSpan, + type ErrorDetailTrace, + type ErrorDetailOutput, +} from "./error-detail" export { diagnoseService } from "./diagnose-service" export { searchLogs } from "./search-logs" export { mineLogPatterns, clusterLogPatterns, type ClusterableLogRow } from "./mine-log-patterns" diff --git a/packages/query-engine/src/observability/row-mappers.ts b/packages/query-engine/src/observability/row-mappers.ts index 897afd187..8c9c101c0 100644 --- a/packages/query-engine/src/observability/row-mappers.ts +++ b/packages/query-engine/src/observability/row-mappers.ts @@ -36,6 +36,7 @@ export const toLogEntry = (l: ListLogsOutput): LogEntry => ({ export const toErrorSummary = (e: ErrorsByTypeOutput): ErrorSummary => ({ fingerprintHash: e.fingerprintHash, label: e.errorLabel, + sampleMessage: e.sampleMessage ?? "", count: e.count, affectedServicesCount: e.affectedServicesCount, lastSeen: e.lastSeen, diff --git a/packages/query-engine/src/observability/types.ts b/packages/query-engine/src/observability/types.ts index 8e96f8735..471a33ae9 100644 --- a/packages/query-engine/src/observability/types.ts +++ b/packages/query-engine/src/observability/types.ts @@ -100,6 +100,8 @@ export interface ErrorSummary { readonly fingerprintHash: string /** Human-readable display label derived at ingest. */ readonly label: string + /** One occurrence's status message — what tells two fingerprints with the same label apart. */ + readonly sampleMessage: string readonly count: number readonly affectedServicesCount: number readonly lastSeen: string @@ -109,6 +111,9 @@ export interface FindErrorsInput { readonly timeRange: TimeRange readonly service?: string readonly environment?: string + /** "unexpected": only identities outside `namespacePrefix` plus the 5xx/unexpected-envelope markers. */ + readonly identity?: "unexpected" + readonly namespacePrefix?: string readonly limit?: number }