From ab8364e7f632d7ce0e8bc12be2422c1b2e94d93d Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 15 Jul 2026 12:38:37 -0400 Subject: [PATCH] Project tool observability into reviewer transcripts --- .../execute-native-agent-task.mjs | 60 ++++++++++++ .../prepare-agent-task-upload.mjs | 52 ++++++++++- .../runtime-core/src/agent-task-run-result.ts | 2 + .../runtime-core/src/tool-observability.ts | 92 +++++++++++++++++++ tests/agent-task-contracts.test.ts | 35 +++++++ tests/runtime-sources-materialization.test.ts | 11 ++- 6 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 packages/runtime-core/src/tool-observability.ts diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index 25387a23..e650e5e0 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -139,6 +139,56 @@ function record(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {} } +function canonicalToolObservability(metadata) { + const source = record(record(record(metadata).agents_api).tool_observability) + if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > 64) return undefined + const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean) + return calls.length ? { version: 1, calls } : undefined +} + +function projectCanonicalToolCall(value) { + const call = record(value) + const argumentsSummary = record(call.arguments) + const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : [] + if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1 + || !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name) + || !["succeeded", "failed", "rejected", "pending"].includes(call.status) + || argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0 + || argumentsSummary.count !== keys.length || keys.length > 32 || !keys.every(safeToolIdentifier)) return undefined + const resultSummary = projectCanonicalToolResult(call.result) + if (call.result !== undefined && !resultSummary) return undefined + return Object.fromEntries(Object.entries({ + sequence: call.sequence, turn: call.turn, tool_call_id: call.tool_call_id, tool_name: call.tool_name, status: call.status, + arguments: { keys, count: argumentsSummary.count, redacted: true }, result: resultSummary, + error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." } + : call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." } : undefined, + }).filter(([, item]) => item !== undefined)) +} + +function projectCanonicalToolResult(value) { + const result = record(value) + if (Object.keys(result).length === 0) return undefined + if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined + if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined + return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined +} + +function safeToolIdentifier(value) { + return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value) +} + +function removeRawToolObservability(metadata) { + const visit = (value) => { + if (Array.isArray(value)) return value.forEach(visit) + const entry = record(value) + if (!Object.keys(entry).length) return + const agentsApi = record(record(entry.metadata).agents_api) + delete agentsApi.tool_observability + Object.values(entry).forEach(visit) + } + visit(metadata) +} + function string(value) { return typeof value === "string" ? value.trim() : "" } @@ -519,7 +569,17 @@ const nativeRuntimeResult = request.run_agent && !request.dry_run : {} await rm(nativeResultPath, { force: true }) reviewerEvidence = await canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath) +const toolObservability = canonicalToolObservability(record(nativeRuntimeResult).metadata) + ?? canonicalToolObservability(record(record(nativeRuntimeResult).agent_task_run_result).metadata) let runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization) +removeRawToolObservability(runtimeResult) +const normalizedAgentTaskResult = record(record(runtimeResult).agent_task_run_result) +if (toolObservability) { + normalizedAgentTaskResult.metadata = { + ...record(normalizedAgentTaskResult.metadata), + tool_observability: toolObservability, + } +} assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization) let workspaceApply = { status: "no-op", changedFiles: [] } let runnerWorkspaceCore = null diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index f3b9fa89..368057ce 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -8,6 +8,7 @@ import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson } from "./runtime const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024 const MAX_TRANSCRIPT_EXECUTIONS = 64 const MAX_REVIEW_TEXT_BYTES = 32 * 1024 +const MAX_TOOL_ARGUMENT_KEYS = 32 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd()) const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload")) const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json")) @@ -208,15 +209,64 @@ function projectParsed(value) { const results = Array.isArray(parsed.tool_results) ? parsed.tool_results : Array.isArray(parsed.toolResults) ? parsed.toolResults : [] const errors = Array.isArray(parsed.errors) ? parsed.errors : parsed.error ? [parsed.error] : [] const agent = record(parsed.agent ?? parsed.agent_metadata) + const toolObservability = canonicalToolObservability(parsed.metadata) + ?? canonicalToolObservability(record(parsed.agent_runtime).result?.metadata) return Object.fromEntries(Object.entries({ - agent: Object.fromEntries(["id", "name", "model", "provider", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])), + agent: Object.fromEntries(["id", "name", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])), model_messages: messages.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((message) => boundedText(record(message).content ?? record(message).text ?? message) ? [{ role: boundedText(record(message).role), content: boundedText(record(message).content ?? record(message).text ?? message) }] : []), tool_calls: tools.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall), tool_results: results.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall), errors: errors.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((error) => boundedText(record(error).message ?? error) ? [boundedText(record(error).message ?? error)] : []), + tool_observability: toolObservability, }).filter(([, item]) => Array.isArray(item) ? item.length > 0 : Object.keys(item).length > 0)) } +// This is deliberately limited to the public Agents API summary. Tool payloads +// and provider-specific tool records are not inputs to reviewer artifacts. +function canonicalToolObservability(metadata) { + const source = record(record(record(metadata).agents_api).tool_observability) + if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TRANSCRIPT_EXECUTIONS) return undefined + const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean) + return calls.length ? { version: 1, calls } : undefined +} + +function projectCanonicalToolCall(value) { + const call = record(value) + const argumentsSummary = record(call.arguments) + const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : [] + if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1 + || !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name) + || !["succeeded", "failed", "rejected", "pending"].includes(call.status) + || argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0 + || argumentsSummary.count !== keys.length || keys.length > MAX_TOOL_ARGUMENT_KEYS || !keys.every(safeToolIdentifier)) return undefined + const resultSummary = projectCanonicalToolResult(call.result) + if (call.result !== undefined && !resultSummary) return undefined + return Object.fromEntries(Object.entries({ + sequence: call.sequence, + turn: call.turn, + tool_call_id: call.tool_call_id, + tool_name: call.tool_name, + status: call.status, + arguments: { keys, count: argumentsSummary.count, redacted: true }, + result: resultSummary, + error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." } + : call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." } + : undefined, + }).filter(([, item]) => item !== undefined)) +} + +function projectCanonicalToolResult(value) { + const result = record(value) + if (Object.keys(result).length === 0) return undefined + if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined + if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined + return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined +} + +function safeToolIdentifier(value) { + return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value) +} + async function trustedTranscriptFile(path) { const root = await realpath(artifactsPath).catch(() => "") if (!root) return { unavailable: "artifact-root-missing" } diff --git a/packages/runtime-core/src/agent-task-run-result.ts b/packages/runtime-core/src/agent-task-run-result.ts index f86a46c4..174ee795 100644 --- a/packages/runtime-core/src/agent-task-run-result.ts +++ b/packages/runtime-core/src/agent-task-run-result.ts @@ -2,6 +2,7 @@ import { isPlainObject, numberValue, objectValue, stringValue, stripUndefined } import { normalizeAgentTerminalResult, type AgentTerminalResult } from "./agent-terminal-result.js" import { RUNTIME_ACCESS_SCHEMA, normalizeRuntimeAccess, type RuntimeAccess } from "./runtime-boundary-contracts.js" import { normalizeAgentTaskStatus } from "./status-taxonomy.js" +import { normalizeToolObservability } from "./tool-observability.js" export const AGENT_TASK_RUN_RESULT_SCHEMA = "wp-codebox/agent-task-run-result/v1" as const @@ -137,6 +138,7 @@ export function normalizeAgentTaskRunResult(raw: unknown, options: AgentTaskRunR provider_error: objectValue(result.provider_error), timeout: result.timeout === true ? true : undefined, failure_evidence: objectValue(result.failure_evidence), + tool_observability: normalizeToolObservability(result.metadata) ?? normalizeToolObservability(agentResult.metadata), }), terminal_result: terminalResult, runtime_access: agentTaskRuntimeAccess(result), diff --git a/packages/runtime-core/src/tool-observability.ts b/packages/runtime-core/src/tool-observability.ts new file mode 100644 index 00000000..4ef2825c --- /dev/null +++ b/packages/runtime-core/src/tool-observability.ts @@ -0,0 +1,92 @@ +import { isPlainObject, objectValue, stripUndefined } from "./object-utils.js" + +export const TOOL_OBSERVABILITY_VERSION = 1 as const +const MAX_TOOL_CALLS = 64 +const MAX_IDENTIFIER_LENGTH = 256 +const MAX_ARGUMENT_KEYS = 32 +const RESULT_TYPES = ["array", "object", "string", "integer", "double", "boolean", "null"] as const +type ResultType = typeof RESULT_TYPES[number] + +export interface ToolObservabilityCall { + sequence: number + turn: number + tool_call_id: string + tool_name: string + status: "succeeded" | "failed" | "rejected" | "pending" + arguments: { keys: string[], count: number, redacted: true } + result?: { type: ResultType, count?: number, size?: number } + error?: { code: string, message: string } +} + +export interface ToolObservability { + version: typeof TOOL_OBSERVABILITY_VERSION + calls: ToolObservabilityCall[] +} + +/** Projects the public Agents API lifecycle without retaining tool payloads. */ +export function normalizeToolObservability(metadata: unknown): ToolObservability | undefined { + const observability = objectValue(objectValue(metadata).agents_api).tool_observability + const source = objectValue(observability) + if (source.version !== TOOL_OBSERVABILITY_VERSION || !Array.isArray(source.calls) || source.calls.length > MAX_TOOL_CALLS) return undefined + + const calls = source.calls.map(projectCall).filter((call): call is ToolObservabilityCall => call !== undefined) + return calls.length > 0 ? { version: TOOL_OBSERVABILITY_VERSION, calls } : undefined +} + +function projectCall(value: unknown): ToolObservabilityCall | undefined { + if (!isPlainObject(value)) return undefined + const call = objectValue(value) + const status = call.status + const argumentsSummary = objectValue(call.arguments) + const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : [] + const count = argumentsSummary.count + if (!isPositiveInteger(call.sequence) || !isPositiveInteger(call.turn) + || !safeIdentifier(call.tool_call_id) || !safeIdentifier(call.tool_name) + || !["succeeded", "failed", "rejected", "pending"].includes(String(status)) + || argumentsSummary.redacted !== true || !isNonNegativeInteger(count) || count !== keys.length || keys.length > MAX_ARGUMENT_KEYS + || !keys.every(safeIdentifier)) return undefined + + const result = projectResult(call.result) + if (call.result !== undefined && !result) return undefined + return stripUndefined({ + sequence: call.sequence, + turn: call.turn, + tool_call_id: call.tool_call_id, + tool_name: call.tool_name, + status: status as ToolObservabilityCall["status"], + arguments: { keys, count, redacted: true }, + result, + error: status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." } + : status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." } + : undefined, + }) as ToolObservabilityCall +} + +function projectResult(value: unknown): ToolObservabilityCall["result"] | undefined { + const result = objectValue(value) + if (Object.keys(result).length === 0) return undefined + if (!isResultType(result.type)) return undefined + if (result.type === "array" || result.type === "object") { + return isNonNegativeInteger(result.count) ? { type: result.type, count: result.count } : undefined + } + if (result.type === "string") { + return isNonNegativeInteger(result.size) ? { type: result.type, size: result.size } : undefined + } + return { type: result.type } +} + +function safeIdentifier(value: unknown): value is string { + return typeof value === "string" && value.length <= MAX_IDENTIFIER_LENGTH && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value) +} + +function isResultType(value: unknown): value is ResultType { + return typeof value === "string" && RESULT_TYPES.includes(value as ResultType) +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 +} diff --git a/tests/agent-task-contracts.test.ts b/tests/agent-task-contracts.test.ts index 25392214..9744bac8 100644 --- a/tests/agent-task-contracts.test.ts +++ b/tests/agent-task-contracts.test.ts @@ -18,6 +18,41 @@ assert.equal(succeeded.schema, AGENT_TASK_RUN_RESULT_SCHEMA) assert.equal(succeeded.status, "succeeded") assert.equal(agentTaskRunExitCode({ success: true, agent_task_run_result: succeeded }), 0) +const toolObservability = normalizeAgentTaskRunResult({ + success: true, + metadata: { + agents_api: { + tool_observability: { + version: 1, + calls: [ + { sequence: 1, turn: 1, tool_call_id: "call-success", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 1, redacted: true }, result: { type: "object", count: 2 }, result_body: "secret-sentinel" }, + { sequence: 2, turn: 1, tool_call_id: "call-failure", tool_name: "workspace.write", status: "failed", arguments: { keys: [], count: 0, redacted: true }, error: { code: "raw-secret", message: "secret-sentinel" } }, + { sequence: 3, turn: 2, tool_call_id: "call-rejected", tool_name: "workspace.delete", status: "rejected", arguments: { keys: ["path"], count: 1, redacted: true } }, + { sequence: 4, turn: 2, tool_call_id: "call-pending", tool_name: "workspace.grep", status: "pending", arguments: { keys: ["query"], count: 1, redacted: true } }, + { sequence: 5, turn: 2, tool_call_id: "malformed", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 2, redacted: true } }, + { sequence: 6, turn: 2, tool_call_id: "untrusted secret sentinel", tool_name: "workspace.read", status: "pending", arguments: { keys: [], count: 0, redacted: true } }, + { sequence: 7, turn: 2, tool_call_id: "call-secret-name", tool_name: "untrusted secret sentinel", status: "pending", arguments: { keys: [], count: 0, redacted: true } }, + { sequence: 8, turn: 2, tool_call_id: "call-secret-key", tool_name: "workspace.read", status: "pending", arguments: { keys: ["untrusted secret sentinel"], count: 1, redacted: true } }, + { sequence: 9, turn: 2, tool_call_id: "call-secret-type", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: { type: "untrusted secret sentinel", count: 1 } }, + ], + }, + }, + }, +}) +assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls.map((call: any) => call.status), ["succeeded", "failed", "rejected", "pending"]) +assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[0].result, { type: "object", count: 2 }) +assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[1].error, { code: "tool_call_failed", message: "Tool call failed." }) +assert.equal(JSON.stringify(toolObservability).includes("secret-sentinel"), false, "normalized tool observability never retains payloads or raw errors") +assert.equal(JSON.stringify(toolObservability).includes("untrusted secret sentinel"), false, "normalized tool observability rejects non-identifier summary strings") +for (const resultShape of [{ type: "string", size: 12 }, { type: "integer" }]) { + const normalized = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: 1, calls: [{ sequence: 1, turn: 1, tool_call_id: "call-shape", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: resultShape }] } } } }) + assert.deepEqual((normalized.metadata.tool_observability as any)?.calls[0].result, resultShape) +} +for (const unsupported of [0, 2, "1"]) { + const result = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: unsupported, calls: [{ sequence: 1, turn: 1, tool_call_id: "call", tool_name: "tool", status: "pending", arguments: { keys: [], count: 0, redacted: true } }] } } } }) + assert.equal("tool_observability" in result.metadata, false, "only canonical version 1 tool observability is consumed") +} + const resultFileDirectory = mkdtempSync(join(tmpdir(), "wp-codebox-agent-task-result-file-")) const resultFilePath = join(resultFileDirectory, "result.json") const atomicResult = { schema: "wp-codebox/agent-task-run/v1", success: true, status: "succeeded", padding: "x".repeat(40 * 1024) } diff --git a/tests/runtime-sources-materialization.test.ts b/tests/runtime-sources-materialization.test.ts index 3de522e7..1b304ca1 100644 --- a/tests/runtime-sources-materialization.test.ts +++ b/tests/runtime-sources-materialization.test.ts @@ -166,7 +166,7 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => { await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }) assert.match(await readFile(join(upload, ".codebox", "agent-task-artifacts", "safe.json"), "utf8"), /OpenAiProvider/) await mkdir(join(artifacts, "files"), { recursive: true }) - const transcriptSource = JSON.stringify({ schema: "wp-codebox/agent-transcript/v1", executions: [{ executionIndex: 0, command: "wp-codebox.agent-sandbox-run", exitCode: 1, stderr: "Repeated workspace error", parsed: { agent: { id: "fixture", provider: "openai" }, messages: [{ role: "assistant", content: "Target snippet: { assert.match(transcript, /workspace\/src\/plugin.php/) assert.match(transcript, /"bytes": 34/) assert.match(transcript, /"sha256": "[a-f0-9]{64}"/) - assert.doesNotMatch(transcript, /private-runtime-source|secret-transcript-value|\/Users\/example/) + assert.match(transcript, /"tool_observability"/) + assert.match(transcript, /"tool_call_failed"/) + assert.match(transcript, /"type": "string"/) + assert.match(transcript, /"type": "integer"/) + assert.doesNotMatch(transcript, /raw-error|"args"|"provider"|"hash"/) + assert.doesNotMatch(transcript, /private-runtime-source|secret-transcript-value|untrusted secret sentinel|\/Users\/example/) + const stagedWorkflowResult = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8") + assert.match(stagedWorkflowResult, /"projection_sha256": "[a-f0-9]{64}"/, "the staged result records the digest of the sanitized transcript projection") const transcriptExclusions = await readFile(join(upload, ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8") assert.match(transcriptExclusions, /canonical_transcripts/) assert.match(transcriptExclusions, /codebox-transcript/)