Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions apps/api/src/chat/loop/turn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,9 @@ const runStep = (
? annotateModelCallEnd(collected)
: Effect.void,
),
// Mapped before the span closes so it records `@maple/llm/LlmCallError`
// with a reason, not the package's own `AI.Error` tag.
Stream.mapError((error) => toLlmCallError("chat.turn", error)),
// One span per model call — an attempt, not a logical step, because a
// retry costs the same money and wall clock and deserves its own record.
// The catch below sits outside, so a failed call ends this span with the
Expand Down Expand Up @@ -368,9 +371,8 @@ const runStep = (
// A model failure either retries the step or ends the turn as a recorded event. Either
// way it does not kill the stream: the session log is durable, so a client reconnecting
// after the failure must still be able to read what happened.
Stream.catch((error) => {
Stream.catch((called) => {
failed = true
const called = toLlmCallError("chat.turn", error)

// Aborted mid-stream. The DO already recorded the terminal event when it cleared the
// claim, so emitting anything here would be a second one.
Expand Down
6 changes: 5 additions & 1 deletion apps/api/src/chat/turn-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,9 @@ const compactIfNeeded = (
usage: TurnUsage,
): Effect.Effect<void, never, LLMClientService> =>
Effect.gen(function* () {
const { contextLimitOf, outputLimitOf } = yield* Effect.promise(() => import("../platform/Llm"))
const { contextLimitOf, outputLimitOf, toLlmCallError } = yield* Effect.promise(
() => import("../platform/Llm"),
)
const {
addUsage,
isNearContextLimit,
Expand Down Expand Up @@ -190,6 +192,8 @@ const compactIfNeeded = (
})
const response = yield* LLM.generate(request).pipe(
Effect.tap((generated) => annotateModelResponse(generated)),
// Mapped inside the span so a failed compaction records Maple's tag, not `AI.Error`.
Effect.mapError((error) => toLlmCallError("chat.compaction", error)),
Effect.withSpan(modelCallSpanName(model), {
kind: "client",
// The whole request, so the span records what the model was actually
Expand Down
4 changes: 2 additions & 2 deletions apps/api/src/mcp/lib/dashboard-schema-doc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -500,8 +500,8 @@ const rawSqlSection = (): string =>
"",
"```sql",
"SELECT toStartOfInterval(Timestamp, INTERVAL $__interval_s SECOND) AS bucket,",
" countIf(SeverityText = 'Error') AS Error,",
" countIf(SeverityText = 'Warn') AS Warn",
" countIf(SeverityNumber BETWEEN 17 AND 20) AS Error,",
" countIf(SeverityNumber BETWEEN 13 AND 16) AS Warn",
"FROM logs",
"WHERE $__orgFilter AND $__timeFilter(Timestamp)",
"GROUP BY bucket",
Expand Down
22 changes: 22 additions & 0 deletions apps/api/src/mcp/lib/render-trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,25 @@ describe("renderTraceOverview", () => {
expect(text).toContain("span:deadbeef") // short span ref for the error log
})
})

describe("renderTraceOverview errorsOnly", () => {
it("says which policy pruned the tree", () => {
const spans = [
span("root", {
children: [span("ok"), span("bad", { statusCode: "Error", statusMessage: "boom" })],
}),
]
const { lines, overview } = renderTraceOverview({
...base,
spanCount: 3,
spans,
budget: 100,
options: { errorsOnly: true },
})
const text = lines.join("\n")
expect(overview.renderedCount).toBe(2)
expect(text).toContain("error spans and their ancestors only")
expect(text).toContain("bad")
expect(text).not.toContain("ok —")
})
})
10 changes: 7 additions & 3 deletions apps/api/src/mcp/lib/render-trace.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Array as Arr, pipe } from "effect"
import type { SpanNode } from "@maple/query-engine/observability"
import { formatDurationFromMs, truncate } from "./format"
import { selectOverviewSpans, type OverviewSelection } from "./span-tree"
import { selectOverviewSpans, type OverviewOptions, type OverviewSelection } from "./span-tree"

export interface TraceOverviewLog {
readonly timestamp: string
Expand All @@ -20,6 +20,7 @@ export interface TraceOverviewInput {
readonly logs: ReadonlyArray<TraceOverviewLog>
/** Max spans to render before collapsing the rest (see `selectOverviewSpans`). */
readonly budget: number
readonly options?: OverviewOptions
}

export interface RenderedTraceOverview {
Expand All @@ -33,16 +34,19 @@ export interface RenderedTraceOverview {
* collapse markers — is unit-testable without a live warehouse.
*/
export function renderTraceOverview(input: TraceOverviewInput): RenderedTraceOverview {
const overview = selectOverviewSpans(input.spans, input.budget)
const overview = selectOverviewSpans(input.spans, input.budget, input.options)

const lines: string[] = [
`## Trace ${input.traceId} (${input.serviceCount} services, ${input.spanCount} spans, ${formatDurationFromMs(input.rootDurationMs)})`,
``,
]

if (overview.truncated) {
const policy = input.options?.errorsOnly
? "error spans and their ancestors only"
: "errors and longest first"
lines.push(
`_Showing ${overview.renderedCount} of ${input.spanCount} spans (errors and longest first). Use \`inspect_span trace_id="${input.traceId}" span_id="…"\` for one span's full attributes, or \`search_traces\` to find more._`,
`_Showing ${overview.renderedCount} of ${input.spanCount} spans (${policy}). Use \`inspect_span trace_id="${input.traceId}" span_id="…"\` for one span's full attributes, or \`search_traces\` to find more._`,
``,
)
}
Expand Down
17 changes: 17 additions & 0 deletions apps/api/src/mcp/lib/span-tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,20 @@ describe("selectOverviewSpans", () => {
result.roots.forEach(checkConnected)
})
})

describe("selectOverviewSpans errorsOnly", () => {
it("keeps only errors, their ancestors and the roots, even when the trace fits the budget", () => {
const roots = bigTree()
const result = selectOverviewSpans(roots, 100, { errorsOnly: true })
const kept = ids(result.roots)
expect(kept.has("root")).toBe(true)
expect(kept.has("a")).toBe(true)
expect(kept.has("a2")).toBe(true)
expect(kept.has("a1")).toBe(false)
expect(kept.has("b")).toBe(false)
expect(kept.has("c")).toBe(false)
expect(result.renderedCount).toBe(3)
expect(result.truncated).toBe(true)
expect(result.omittedByParent.get("root")?.count).toBe(12)
})
})
28 changes: 21 additions & 7 deletions apps/api/src/mcp/lib/span-tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,27 @@ function subtreeSize(node: SpanNode): number {
* long/structural spans. Returns the original tree unchanged when it already
* fits within `budget` (so small traces render exactly as before).
*/
export function selectOverviewSpans(roots: ReadonlyArray<SpanNode>, budget: number): OverviewSelection {
export interface OverviewOptions {
/**
* Keep only error spans, their ancestor chains and the roots — no budget fill. Applies even
* to traces that would fit the budget, since the point is to strip the healthy spans.
*/
readonly errorsOnly?: boolean
}

export function selectOverviewSpans(
roots: ReadonlyArray<SpanNode>,
budget: number,
options: OverviewOptions = {},
): OverviewSelection {
let totalCount = 0
const parentOf = new Map<string, SpanNode | null>()
forEachNode(roots, (node, parent) => {
totalCount += 1
parentOf.set(key(node), parent)
})

if (totalCount <= budget) {
if (totalCount <= budget && !options.errorsOnly) {
return {
roots: roots as SpanNode[],
renderedCount: totalCount,
Expand Down Expand Up @@ -106,10 +118,12 @@ export function selectOverviewSpans(roots: ReadonlyArray<SpanNode>, budget: numb
})

// 2. Fill remaining budget by score, highest first.
candidates.sort((a, b) => score(b) - score(a))
for (const node of candidates) {
if (selected.size >= budget) break
addWithAncestors(node)
if (!options.errorsOnly) {
candidates.sort((a, b) => score(b) - score(a))
for (const node of candidates) {
if (selected.size >= budget) break
addWithAncestors(node)
}
}

// 3. Rebuild a pruned tree of new nodes; record omissions per parent.
Expand Down Expand Up @@ -138,7 +152,7 @@ export function selectOverviewSpans(roots: ReadonlyArray<SpanNode>, budget: numb
roots: prunedRoots,
renderedCount: selected.size,
totalCount,
truncated: true,
truncated: selected.size < totalCount,
omittedByParent,
}
}
Expand Down
25 changes: 21 additions & 4 deletions apps/api/src/mcp/tools/error-detail.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,20 @@ export function registerErrorDetailTool(server: McpToolRegistrar) {
` Services: ${t.services.join(", ")}`,
` Time: ${t.startTime}`,
)
if (t.errorMessage) {
// The failing span first: name, service, status and the attributes that say what it
// was doing. That is the line an investigator needs; the logs below are context.
if (t.errorSpan) {
lines.push(
` Error span: ${t.errorSpan.name} — ${t.errorSpan.serviceName} span=${t.errorSpan.spanId}`,
)
if (t.errorSpan.statusMessage) {
lines.push(` Status: "${truncate(t.errorSpan.statusMessage, 160)}"`)
}
const attrs = Object.entries(t.errorSpan.attributes)
if (attrs.length > 0) {
lines.push(` {${attrs.map(([k, v]) => `${k}=${truncate(v, 60)}`).join(", ")}}`)
}
} else if (t.errorMessage) {
lines.push(` Error: ${truncate(t.errorMessage, 120)}`)
}
if (t.logs.length > 0) {
Expand All @@ -137,9 +150,10 @@ export function registerErrorDetailTool(server: McpToolRegistrar) {
lines.push(``)
}

const nextSteps = Arr.map(
Arr.take(result.traces, 3),
(t) => `\`inspect_trace trace_id="${t.traceId}"\` — full span tree`,
const nextSteps = Arr.map(Arr.take(result.traces, 3), (t) =>
t.errorSpan
? `\`inspect_span trace_id="${t.traceId}" span_id="${t.errorSpan.spanId}"\` — the failing span's full attributes`
: `\`inspect_trace trace_id="${t.traceId}" errors_only=true\` — the failing spans only`,
)
nextSteps.push(
`\`search_logs service="${service ?? ""}" severity="ERROR"\` — search for related error logs`,
Expand All @@ -160,6 +174,9 @@ export function registerErrorDetailTool(server: McpToolRegistrar) {
services: [...t.services],
startTime: t.startTime,
errorMessage: t.errorMessage || undefined,
errorSpan: t.errorSpan
? { ...t.errorSpan, attributes: { ...t.errorSpan.attributes } }
: undefined,
logs: Arr.map(t.logs, (l) => ({ ...l })),
})),
},
Expand Down
53 changes: 46 additions & 7 deletions apps/api/src/mcp/tools/find-errors.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
import { optionalNumberParam, optionalStringParam, optionalTimeParam, type McpToolRegistrar } from "./types"
import {
optionalNumberParam,
optionalStringParam,
optionalTimeParam,
validationError,
type McpToolRegistrar,
} from "./types"
import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error"
import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse"
import { resolveTimeRange } from "@/mcp/lib/time"
import { formatNumber, formatTable } from "@/mcp/lib/format"
import { formatNumber, formatTable, truncate } from "@/mcp/lib/format"
import { formatNextSteps } from "@/mcp/lib/next-steps"
import { Array as Arr, Effect, Schema } from "effect"
import { createDualContent } from "@/mcp/lib/structured-output"
Expand All @@ -22,31 +28,62 @@ export function registerFindErrorsTool(server: McpToolRegistrar) {
end_time: optionalTimeParam("End of time range (YYYY-MM-DD HH:mm:ss)"),
service: optionalStringParam("Filter to a specific service"),
environment: optionalStringParam("Filter by deployment environment (e.g. production, staging)"),
identity: optionalStringParam(
'Pass "unexpected" to keep only identities that break a no-unknown-errors policy: labels outside `namespace_prefix` (library tags such as `AI.Error`, bare `Error`) plus the 5xx and unexpected-error-envelope markers. Omit for all errors.',
),
namespace_prefix: optionalStringParam(
'The prefix every deliberate, namespaced error tag starts with (default "@maple/"). Only used with identity="unexpected".',
),
limit: optionalNumberParam("Max results (default 20)"),
}),
Effect.fn("McpTool.findErrors")(function* ({ start_time, end_time, service, environment, limit }) {
Effect.fn("McpTool.findErrors")(function* ({
start_time,
end_time,
service,
environment,
identity,
namespace_prefix,
limit,
}) {
if (identity !== undefined && identity !== "unexpected") {
return validationError(`Invalid identity: '${identity}'. Pass "unexpected" or omit it.`)
}
const { st, et } = resolveTimeRange(start_time, end_time)
const tenant = yield* CurrentMcpTenant
yield* Effect.annotateCurrentSpan({
orgId: tenant.orgId,
service: service ?? "all",
identity: identity ?? "all",
})

const errors = yield* findErrors({
timeRange: { startTime: st, endTime: et },
service: service ?? undefined,
environment: environment ?? undefined,
identity: identity ?? undefined,
namespacePrefix: namespace_prefix ?? undefined,
limit: limit ?? 20,
}).pipe(
provideWarehouseExecutorFromTenant(tenant),
Effect.mapError(toMcpQueryError("errors_by_type")),
)

if (errors.length === 0) {
return { content: [{ type: "text", text: `No errors found in ${st} — ${et}` }] }
const scope = identity === "unexpected" ? "unexpected-identity errors" : "errors"
return { content: [{ type: "text", text: `No ${scope} found in ${st} — ${et}` }] }
}

const lines: string[] = [`## Errors by Type`, ``]
const lines: string[] = [
identity === "unexpected" ? `## Unexpected Error Identities` : `## Errors by Type`,
``,
]

const headers = ["Error", "Fingerprint", "Count", "Affected Services", "Last Seen"]
// One occurrence's message per row: the same tag can own a dozen fingerprints, and the
// label alone gave no way to tell them apart short of an error_detail call each.
const headers = ["Error", "Message", "Fingerprint", "Count", "Affected Services", "Last Seen"]
const rows = Arr.map(errors, (e) => [
e.label.length > 60 ? e.label.slice(0, 57) + "..." : e.label,
truncate(e.label, 60),
truncate(e.sampleMessage.replace(/\s+/g, " ").replace(/\|/g, "\\|"), 80),
e.fingerprintHash,
formatNumber(e.count),
String(e.affectedServicesCount),
Expand All @@ -72,9 +109,11 @@ export function registerFindErrorsTool(server: McpToolRegistrar) {
tool: "find_errors",
data: {
timeRange: { start: st, end: et },
identity: identity ?? "all",
errors: Arr.map(errors, (e) => ({
fingerprintHash: e.fingerprintHash,
label: e.label,
sampleMessage: e.sampleMessage,
count: e.count,
affectedServicesCount: e.affectedServicesCount,
lastSeen: e.lastSeen,
Expand Down
28 changes: 25 additions & 3 deletions apps/api/src/mcp/tools/inspect-trace.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import { requiredStringParam, optionalStringParam, type McpToolRegistrar } from "./types"
import {
optionalBooleanParam,
optionalNumberParam,
optionalStringParam,
requiredStringParam,
type McpToolRegistrar,
} from "./types"
import { clampLimit } from "@/mcp/lib/limits"
import { warehouseToMcpHandlers } from "@/mcp/lib/map-warehouse-error"
import { withTenantExecutor } from "@/mcp/lib/query-warehouse"
import { formatNextSteps } from "@/mcp/lib/next-steps"
Expand All @@ -14,6 +21,8 @@ import { inspectTrace, type SpanNode } from "@maple/query-engine/observability"
* to this budget — deeper inspection goes through `inspect_span` / `search_traces`.
*/
const MAX_OVERVIEW_SPANS = 100
/** Hard ceiling for `max_spans`; past this a single response stops being readable. */
const MAX_OVERVIEW_SPANS_CEILING = 300

export function registerInspectTraceTool(server: McpToolRegistrar) {
server.tool(
Expand All @@ -24,9 +33,20 @@ export function registerInspectTraceTool(server: McpToolRegistrar) {
timestamp: optionalStringParam(
"ISO-8601 timestamp of any span in the trace (e.g. from `search_traces` results). Used to narrow the ClickHouse scan to a ±1h window — required for traces older than 24h, strongly recommended otherwise.",
),
errors_only: optionalBooleanParam(
"Render only error spans, their ancestors and the roots — the fastest way to read a large trace's failure without its healthy spans.",
),
max_spans: optionalNumberParam(
`Spans to render before collapsing the rest (default ${MAX_OVERVIEW_SPANS}, max ${MAX_OVERVIEW_SPANS_CEILING}). Errors and roots are always kept.`,
),
}),
Effect.fn("McpTool.inspectTrace")(function* ({ trace_id, timestamp }) {
Effect.fn("McpTool.inspectTrace")(function* ({ trace_id, timestamp, errors_only, max_spans }) {
yield* Effect.annotateCurrentSpan("traceId", trace_id)
const budget = clampLimit(max_spans, {
defaultValue: MAX_OVERVIEW_SPANS,
max: MAX_OVERVIEW_SPANS_CEILING,
})
const options = { errorsOnly: errors_only === true }

const timestampHint = timestamp ? new Date(timestamp) : undefined
if (timestampHint && Number.isNaN(timestampHint.getTime())) {
Expand Down Expand Up @@ -60,12 +80,14 @@ export function registerInspectTraceTool(server: McpToolRegistrar) {
rootDurationMs: result.rootDurationMs,
spans: result.spans,
logs: result.logs,
budget: MAX_OVERVIEW_SPANS,
budget,
options,
})

yield* Effect.annotateCurrentSpan({
"result.rowCount": result.spanCount,
"result.renderedSpanCount": overview.renderedCount,
"result.errorsOnly": options.errorsOnly,
})

const collectServices = (n: SpanNode): string[] => [
Expand Down
Loading
Loading