From ce1a3aed613e98c97a91f8fd9c6dd667dbf38368 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:05:47 +0000 Subject: [PATCH 01/18] Plan deterministic trace graders Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/weekly-network-domains-audit.lock.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-network-domains-audit.lock.yml b/.github/workflows/weekly-network-domains-audit.lock.yml index a274b6de207..d27c362ba8a 100644 --- a/.github/workflows/weekly-network-domains-audit.lock.yml +++ b/.github/workflows/weekly-network-domains-audit.lock.yml @@ -1505,7 +1505,7 @@ jobs: mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.4 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.4 --rootless - name: Install GitHub Copilot CLI run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" env: @@ -1520,6 +1520,7 @@ jobs: id: detection_agentic_execution if: always() && steps.detection_guard.outputs.run_detection == 'true' continue-on-error: true + timeout-minutes: 20 env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE From de6658f4b26809e1ae627720f028b358fa50c0d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:46:32 +0000 Subject: [PATCH 02/18] Add initial deterministic trace graders Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/redact_secrets.cjs | 38 +- actions/setup/js/trace_graders.cjs | 679 ++++++++++++++++++ actions/setup/js/trace_graders.test.cjs | 567 +++++++++++++++ .../sh/prepare_threat_detection_files.sh | 9 + .../content/docs/reference/trace-graders.md | 63 ++ pkg/constants/job_constants.go | 14 + .../compiler_orchestrator_workflow.go | 7 + pkg/workflow/compiler_yaml_artifacts.go | 9 + pkg/workflow/compiler_yaml_graders.go | 227 ++++++ pkg/workflow/compiler_yaml_post_agent.go | 13 + pkg/workflow/frontmatter_types.go | 5 + pkg/workflow/graders_config.go | 414 +++++++++++ pkg/workflow/graders_config_test.go | 381 ++++++++++ pkg/workflow/workflow_data.go | 1 + 14 files changed, 2426 insertions(+), 1 deletion(-) create mode 100644 actions/setup/js/trace_graders.cjs create mode 100644 actions/setup/js/trace_graders.test.cjs create mode 100644 docs/src/content/docs/reference/trace-graders.md create mode 100644 pkg/workflow/compiler_yaml_graders.go create mode 100644 pkg/workflow/graders_config.go create mode 100644 pkg/workflow/graders_config_test.go diff --git a/actions/setup/js/redact_secrets.cjs b/actions/setup/js/redact_secrets.cjs index 6d0d15a37cf..ffbdc86bc79 100644 --- a/actions/setup/js/redact_secrets.cjs +++ b/actions/setup/js/redact_secrets.cjs @@ -343,4 +343,40 @@ async function main() { } } -module.exports = { main, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS }; +/** + * Targeted redaction pass for files in a specific directory. Used by the + * post-graders redaction step to scan grader output files that were written + * after the initial full-workspace redaction pass. + * + * @param {string} dir - Absolute directory path to scan + */ +async function redactFilesInDir(dir) { + try { + const secretNames = (process.env.GH_AW_SECRET_NAMES || "").split(",").filter(n => n.trim()); + /** @type {string[]} */ + const secretValues = []; + for (const secretName of secretNames) { + const value = process.env[`SECRET_${secretName.trim()}`]; + if (typeof value === "string" && value.trim() !== "") { + secretValues.push(value.trim()); + } + } + secretValues.push(...extractMCPGatewayTokens(MCP_GATEWAY_CONFIG_PATHS)); + + const targetExtensions = [".json"]; + const files = findFiles(dir, targetExtensions); + if (files.length === 0) return; + + let totalRedactions = 0; + for (const file of files) { + totalRedactions += processFile(file, secretValues); + } + if (totalRedactions > 0) { + core.info(`Grader output redaction: ${totalRedactions} redaction(s) in ${dir}`); + } + } catch (error) { + core.warning(`Grader output redaction failed: ${getErrorMessage(error)}`); + } +} + +module.exports = { main, redactFilesInDir, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS }; diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs new file mode 100644 index 00000000000..6fca84159b8 --- /dev/null +++ b/actions/setup/js/trace_graders.cjs @@ -0,0 +1,679 @@ +// @ts-check +/// + +const fs = require("fs"); +const path = require("path"); +const vm = require("vm"); +const crypto = require("crypto"); +const { getErrorMessage } = require("./error_helpers.cjs"); + +// --- Constants --- +const TMP_GH_AW = "/tmp/gh-aw"; +const GRADERS_DIR = path.join(TMP_GH_AW, "agent", "graders"); +const MANIFEST_PATH = path.join(GRADERS_DIR, "grader_manifest.json"); +const RESULTS_PATH = path.join(GRADERS_DIR, "grader_results.json"); + +// Trace source file paths +const TOKEN_USAGE_PATHS = [ + path.join(TMP_GH_AW, "sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl"), + path.join(TMP_GH_AW, "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl"), + path.join(TMP_GH_AW, "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl"), +]; +const AGENT_USAGE_PATH = path.join(TMP_GH_AW, "agent_usage.json"); +const MCP_GATEWAY_LOG_PATHS = [path.join(TMP_GH_AW, "mcp-logs/gateway.jsonl"), path.join(TMP_GH_AW, "mcp-logs/mcp-gateway.jsonl")]; +const AGENT_OUTPUT_PATH = path.join(TMP_GH_AW, "agent_output.json"); +const AGENT_LOG_PATH = path.join(TMP_GH_AW, "agent.log"); +const AGENT_LOG_JSONL_PATH = path.join(TMP_GH_AW, "agent_log.jsonl"); + +// Safety limits +const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB +const MAX_LINE_LENGTH = 1024 * 1024; // 1 MB per line +const SCRIPT_TIMEOUT_MS = 5000; // 5 seconds per custom grader + +const GRADER_VERSION = 1; +const IMPLEMENTATION_ID = "gh-aw/trace-graders"; + +// --- Trace preprocessing --- + +/** + * Safely read a file if it exists and is within size limits. + * @param {string} filePath + * @returns {string|null} + */ +function safeReadFile(filePath) { + try { + if (!fs.existsSync(filePath)) return null; + const stat = fs.statSync(filePath); + if (stat.size > MAX_FILE_SIZE) { + core.warning(`Graders: skipping oversized file ${filePath} (${stat.size} bytes)`); + return null; + } + return fs.readFileSync(filePath, "utf-8"); + } catch { + return null; + } +} + +/** + * Safely parse JSONL, skipping malformed or oversized lines. + * @param {string} content + * @returns {object[]} + */ +function safeParseJsonl(content) { + const results = []; + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + if (trimmed.length > MAX_LINE_LENGTH) continue; + try { + results.push(JSON.parse(trimmed)); + } catch { + // skip malformed lines + } + } + return results; +} + +/** + * Safely parse JSON, returning null on failure. + * @param {string} content + * @returns {object|null} + */ +function safeParseJson(content) { + if (content.length > MAX_FILE_SIZE) return null; + try { + return JSON.parse(content); + } catch { + return null; + } +} + +/** + * Read the first available file from a list of candidate paths. + * @param {string[]} paths + * @returns {string|null} + */ +function readFirstAvailable(paths) { + for (const p of paths) { + const content = safeReadFile(p); + if (content !== null) return content; + } + return null; +} + +/** + * Deep-freeze an object recursively. Returns the same object. + * @template T + * @param {T} obj + * @returns {T} + */ +function deepFreeze(obj) { + if (obj === null || typeof obj !== "object") return obj; + Object.freeze(obj); + for (const key of Object.getOwnPropertyNames(obj)) { + const v = /** @type {any} */ (obj)[key]; + if (v !== null && typeof v === "object" && !Object.isFrozen(v)) { + deepFreeze(v); + } + } + return obj; +} + +/** + * Deep clone via JSON round-trip (safe for plain data objects). + * @param {any} obj + * @returns {any} + */ +function deepClone(obj) { + if (obj === null || obj === undefined) return obj; + return JSON.parse(JSON.stringify(obj)); +} + +/** + * @typedef {object} PreprocessedTrace + * @property {object[]} tokenUsageEntries - Parsed token-usage JSONL records + * @property {object|null} agentUsage - Parsed agent_usage.json + * @property {object[]} mcpGatewayEntries - Parsed MCP gateway log records + * @property {object|null} agentOutput - Parsed agent_output.json + * @property {object[]} toolCalls - Extracted tool call records from MCP gateway + * @property {object[]} gatewayRequests - Request/response pairs from gateway + * @property {object[]} retryEvents - Detected retry events + * @property {object[]} errorEvents - Detected error events + * @property {object[]} steps - Extracted execution steps (LLM requests) + * @property {number} totalInputTokens - Sum of input tokens + * @property {number} totalOutputTokens - Sum of output tokens + * @property {number} totalDurationMs - Sum of duration_ms from token usage + * @property {number} totalRequests - Count of token usage entries (LLM requests) + * @property {object[]} files - Files mentioned in agent output + * @property {object[]} artifacts - Artifacts/outputs from agent + */ + +/** + * Single preprocessing pass over all trace files. + * @returns {PreprocessedTrace} + */ +function preprocessTrace() { + // Token usage + const tokenContent = readFirstAvailable(TOKEN_USAGE_PATHS); + const tokenUsageEntries = tokenContent ? safeParseJsonl(tokenContent) : []; + + // Agent usage + const agentUsageContent = safeReadFile(AGENT_USAGE_PATH); + const agentUsage = agentUsageContent ? safeParseJson(agentUsageContent) : null; + + // MCP gateway logs + const gatewayContent = readFirstAvailable(MCP_GATEWAY_LOG_PATHS); + const mcpGatewayEntries = gatewayContent ? safeParseJsonl(gatewayContent) : []; + + // Agent output + const agentOutputContent = safeReadFile(AGENT_OUTPUT_PATH); + const agentOutput = agentOutputContent ? safeParseJson(agentOutputContent) : null; + + // Extract tool calls from MCP gateway entries + const toolCalls = mcpGatewayEntries.filter(e => e.type === "tool_call" || e.method === "tools/call" || e.event === "tool_call"); + + // Gateway request/response pairs + const gatewayRequests = mcpGatewayEntries.filter(e => e.type === "request" || e.type === "response" || e.method); + + // Retry events + const retryEvents = mcpGatewayEntries.filter(e => e.retry === true || e.event === "retry" || (typeof e.message === "string" && /retry|retrying/i.test(String(e.message)))); + + // Error events + const errorEvents = mcpGatewayEntries.filter(e => e.level === "error" || e.type === "error" || e.event === "error"); + + // Aggregate token counts + let totalInputTokens = 0; + let totalOutputTokens = 0; + let totalDurationMs = 0; + for (const entry of tokenUsageEntries) { + totalInputTokens += Number(entry.input_tokens) || 0; + totalOutputTokens += Number(entry.output_tokens) || 0; + totalDurationMs += Number(entry.duration_ms) || 0; + } + + // Steps: each token usage entry represents an LLM request/step + const steps = tokenUsageEntries.map((entry, i) => ({ + index: i, + inputTokens: Number(entry.input_tokens) || 0, + outputTokens: Number(entry.output_tokens) || 0, + durationMs: Number(entry.duration_ms) || 0, + model: entry.model || null, + })); + + // Extract files and artifacts from agent output + const ao = /** @type {any} */ (agentOutput); + const files = ao && Array.isArray(ao.files) ? ao.files : []; + const artifacts = ao && Array.isArray(ao.outputs) ? ao.outputs : ao && Array.isArray(ao.items) ? ao.items : []; + + return { + tokenUsageEntries, + agentUsage, + mcpGatewayEntries, + agentOutput, + toolCalls, + gatewayRequests, + retryEvents, + errorEvents, + steps, + totalInputTokens, + totalOutputTokens, + totalDurationMs, + totalRequests: tokenUsageEntries.length, + files, + artifacts, + }; +} + +// --- Built-in graders --- + +/** @type {Record} */ +const BUILTIN_META = { + "tool-success-rate": { unit: "ratio", direction: "higher_is_better", threshold: 0.8, min: 0, max: 1 }, + "tool-failure-count": { unit: "count", direction: "lower_is_better", threshold: 5 }, + retries: { unit: "count", direction: "lower_is_better", threshold: 10 }, + loops: { unit: "count", direction: "lower_is_better", threshold: 3 }, + "trajectory-efficiency": { unit: "ratio", direction: "higher_is_better", min: 0, max: 1 }, + "execution-step-count": { unit: "count", direction: "lower_is_better" }, + "execution-duration": { unit: "ms", direction: "lower_is_better" }, + "context-growth": { unit: "factor", direction: "lower_is_better" }, + "artifact-production": { unit: "count", direction: "higher_is_better" }, +}; + +/** + * @param {PreprocessedTrace} trace + * @returns {number} Success rate of tool calls (0-1), or 1 if no tool calls + */ +function gradeToolSuccessRate(trace) { + if (trace.toolCalls.length === 0) return 1; + const successes = trace.toolCalls.filter(t => t.success === true || (t.success !== false && t.status !== "error" && t.status !== "failure" && t.error === undefined)).length; + return successes / trace.toolCalls.length; +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeToolFailureCount(trace) { + return trace.toolCalls.filter(t => t.success === false || t.status === "error" || t.status === "failure" || t.error !== undefined).length; +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeRetries(trace) { + return trace.retryEvents.length; +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeLoops(trace) { + let loops = 0; + let prevKey = ""; + for (const t of trace.toolCalls) { + const key = `${String(t.name || t.tool)}:${JSON.stringify(t.arguments || t.params || "")}`; + if (key === prevKey) loops++; + prevKey = key; + } + return loops; +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeTrajectoryEfficiency(trace) { + if (trace.toolCalls.length === 0) return 1; + const uniqueTools = new Set(trace.toolCalls.map(t => String(t.name || t.tool || ""))); + return Math.min(1, uniqueTools.size / trace.toolCalls.length); +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeExecutionStepCount(trace) { return trace.totalRequests; } + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeExecutionDuration(trace) { return trace.totalDurationMs; } + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeContextGrowth(trace) { + if (trace.tokenUsageEntries.length < 2) return 1; + const first = trace.tokenUsageEntries[0]; + const firstTokens = (Number(first.input_tokens) || 0) + (Number(first.output_tokens) || 0); + if (firstTokens === 0) return 1; + const totalTokens = trace.totalInputTokens + trace.totalOutputTokens; + return totalTokens / firstTokens; +} + +/** @param {PreprocessedTrace} trace @returns {number} */ +function gradeArtifactProduction(trace) { + return trace.artifacts.length; +} + +/** @type {Record number>} */ +const BUILTIN_GRADERS = { + "tool-success-rate": gradeToolSuccessRate, + "tool-failure-count": gradeToolFailureCount, + retries: gradeRetries, + loops: gradeLoops, + "trajectory-efficiency": gradeTrajectoryEfficiency, + "execution-step-count": gradeExecutionStepCount, + "execution-duration": gradeExecutionDuration, + "context-growth": gradeContextGrowth, + "artifact-production": gradeArtifactProduction, +}; + +// --- Execution --- + +/** + * Evaluate a grader's pass/fail against its threshold. + * @param {number} value + * @param {string} direction + * @param {number|undefined} threshold + * @returns {boolean|null} null if no threshold set + */ +function evaluateThreshold(value, direction, threshold) { + if (threshold === undefined || threshold === null) return null; + if (direction === "higher_is_better") return value >= threshold; + if (direction === "lower_is_better") return value <= threshold; + return null; +} + +/** + * @typedef {object} GraderResult + * @property {string} id + * @property {string} name + * @property {number|null} value + * @property {string} unit + * @property {boolean|null} passed + * @property {string} status - "pass" | "fail" | "error" | "unavailable" + * @property {string} [severity] + * @property {string} [details] + * @property {string} [message] + * @property {string} [error] + * @property {string} source - "builtin" | "inline" + * @property {{id: string, version: number, digest?: string}} implementation + */ + +/** + * Normalize a grader result from either built-in number or custom object return. + * @param {string} id + * @param {any} rawResult - number or {value, unit?, passed?, severity?, details?, message?} + * @param {{name: string, unit: string, direction: string, threshold?: number, source: string, digest?: string}} meta + * @returns {GraderResult} + */ +function normalizeResult(id, rawResult, meta) { + /** @type {GraderResult} */ + const base = { + id, + name: meta.name || id, + value: null, + unit: meta.unit || "", + passed: null, + status: "error", + source: meta.source, + implementation: { id: IMPLEMENTATION_ID, version: GRADER_VERSION, ...(meta.digest ? { digest: meta.digest } : {}) }, + }; + + if (rawResult === null || rawResult === undefined) { + base.status = "unavailable"; + base.message = "grader returned null/undefined"; + return base; + } + + let value; + if (typeof rawResult === "object" && rawResult !== null && !Array.isArray(rawResult)) { + // Object result from custom script + value = rawResult.value; + if (rawResult.unit) base.unit = String(rawResult.unit); + if (rawResult.severity) base.severity = String(rawResult.severity); + if (rawResult.details) base.details = String(rawResult.details); + if (rawResult.message) base.message = String(rawResult.message); + if (typeof rawResult.passed === "boolean") base.passed = rawResult.passed; + } else { + value = rawResult; + } + + if (typeof value !== "number" || !isFinite(value)) { + base.status = "error"; + base.error = `grader ${id} returned non-finite value: ${value}`; + return base; + } + + base.value = value; + + // Evaluate threshold if not already set by custom script + if (base.passed === null) { + base.passed = evaluateThreshold(value, meta.direction, meta.threshold); + } + + // Determine status + if (base.passed === true) base.status = "pass"; + else if (base.passed === false) base.status = "fail"; + else base.status = "pass"; // no threshold = informational pass + + return base; +} + +/** + * Run a single built-in grader safely. + * @param {string} id + * @param {PreprocessedTrace} trace + * @param {{name: string, unit: string, direction: string, threshold?: number, source: string}} meta + * @returns {GraderResult} + */ +function runBuiltinGrader(id, trace, meta) { + const fn = BUILTIN_GRADERS[id]; + if (!fn) { + return { ...normalizeResult(id, null, meta), status: "error", error: `grader ${id}: no implementation found` }; + } + try { + const value = fn(trace); + return normalizeResult(id, value, meta); + } catch (err) { + const result = normalizeResult(id, null, meta); + result.status = "error"; + result.error = `grader ${id} runtime error: ${getErrorMessage(err)}`; + return result; + } +} + +/** + * Run a custom inline script in a node:vm sandbox. + * Script receives {trace, run, workflow, config, helpers} and should return {value, ...} or a number. + * @param {string} id + * @param {string} script + * @param {PreprocessedTrace} trace + * @param {{name: string, unit: string, direction: string, threshold?: number, source: string, digest?: string, config?: object}} meta + * @returns {GraderResult} + */ +function runCustomGrader(id, script, trace, meta) { + try { + // Build the frozen sandbox context — no require, process, fetch, Date, Math.random + const frozenTrace = deepFreeze(deepClone(trace)); + const runCtx = deepFreeze({ + graderCount: 0, // filled by caller + }); + const workflowCtx = deepFreeze({}); + const config = deepFreeze(deepClone(meta.config || {})); + const helpers = deepFreeze({ + clamp: (/** @type {number} */ v, /** @type {number} */ lo, /** @type {number} */ hi) => Math.max(lo, Math.min(hi, v)), + ratio: (/** @type {number} */ num, /** @type {number} */ den) => (den === 0 ? 0 : num / den), + sum: (/** @type {number[]} */ arr) => arr.reduce((a, b) => a + b, 0), + }); + + // Wrap script as function body + const wrappedScript = `(function(trace, run, workflow, config, helpers) { "use strict"; ${script} })`; + + const sandbox = { + Math: Object.freeze({ ...Math, random: undefined }), + JSON: Object.freeze({ parse: JSON.parse, stringify: JSON.stringify }), + Array, + Object, + String, + Number, + Boolean, + RegExp, + Map, + Set, + isFinite, + isNaN, + parseInt, + parseFloat, + undefined, + NaN, + Infinity, + }; + const ctx = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); + + const fn = vm.runInContext(wrappedScript, ctx, { timeout: SCRIPT_TIMEOUT_MS, filename: `grader:${id}` }); + const rawResult = fn(frozenTrace, runCtx, workflowCtx, config, helpers); + return normalizeResult(id, rawResult, meta); + } catch (err) { + const result = normalizeResult(id, null, meta); + result.status = "error"; + result.error = `grader ${id} runtime error: ${getErrorMessage(err)}`; + return result; + } +} + +/** + * Legacy adapter for existing tests. Runs a grader by id. + * @param {string} id + * @param {boolean} builtin + * @param {string} [script] + * @param {PreprocessedTrace} trace + * @param {object} [config] + * @returns {{ value: number|null, error: string|null }} + */ +function runGrader(id, builtin, script, trace, config) { + const meta = { name: id, unit: "", direction: "", source: builtin ? "builtin" : "inline", config }; + /** @type {GraderResult} */ + let result; + if (builtin && BUILTIN_GRADERS[id]) { + result = runBuiltinGrader(id, trace, meta); + } else if (script) { + result = runCustomGrader(id, script, trace, meta); + } else { + return { value: null, error: `grader ${id}: no implementation found` }; + } + return { value: result.value, error: result.error || null }; +} + +/** + * Main entry point. Called from the github-script step with manifest JSON and base64 exec spec. + * @param {string} manifestJson - JSON string of grader manifest + * @param {string} [execSpecB64] - Base64-encoded JSON array of {id, script} + */ +async function main(manifestJson, execSpecB64) { + /** @type {{version: number, graders: any[]}} */ + let manifest; + try { + manifest = JSON.parse(manifestJson); + } catch (err) { + core.setFailed(`Graders: failed to parse manifest: ${getErrorMessage(err)}`); + return; + } + + // Decode execution spec (custom scripts) + /** @type {Record} */ + const scriptMap = {}; + if (execSpecB64) { + try { + const specJson = Buffer.from(execSpecB64, "base64").toString("utf-8"); + const specs = JSON.parse(specJson); + for (const s of specs) { + if (s.id && s.script) scriptMap[s.id] = s.script; + } + } catch (err) { + core.warning(`Graders: failed to parse exec spec: ${getErrorMessage(err)}`); + } + } + + // Write manifest file + try { + fs.mkdirSync(GRADERS_DIR, { recursive: true }); + fs.writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2)); + } catch (err) { + core.warning(`Graders: failed to write manifest: ${getErrorMessage(err)}`); + } + + // Filter to enabled graders + const graders = manifest.graders || []; + const enabledGraders = graders.filter(g => g.enabled); + if (enabledGraders.length === 0) { + core.info("Graders: no enabled graders, skipping"); + return; + } + + // Single preprocessing pass + core.info(`Graders: preprocessing trace files for ${enabledGraders.length} grader(s)...`); + const trace = preprocessTrace(); + + // Run all graders + /** @type {GraderResult[]} */ + const results = []; + for (const grader of enabledGraders) { + const meta = { + name: grader.name || grader.id, + unit: grader.unit || "", + direction: grader.direction || "", + threshold: grader.threshold, + source: grader.source || "builtin", + digest: grader.digest, + config: grader.config, + }; + /** @type {GraderResult} */ + let result; + if (grader.source === "builtin" && BUILTIN_GRADERS[grader.id]) { + result = runBuiltinGrader(grader.id, trace, meta); + } else if (scriptMap[grader.id]) { + result = runCustomGrader(grader.id, scriptMap[grader.id], trace, meta); + } else { + result = normalizeResult(grader.id, null, meta); + result.status = "unavailable"; + result.error = `grader ${grader.id}: no implementation available`; + } + results.push(result); + if (result.error) { + core.warning(`Grader ${grader.id}: ${result.error}`); + } + } + + // Build normalized output — NO timestamp for deterministic byte-equivalence + const passed = results.filter(r => r.status === "pass").length; + const failed = results.filter(r => r.status === "fail").length; + const errorCount = results.filter(r => r.status === "error").length; + + const output = { + version: GRADER_VERSION, + run: { + graderCount: results.length, + passed, + failed, + errors: errorCount, + }, + results, + }; + + // Write results + try { + fs.writeFileSync(RESULTS_PATH, JSON.stringify(output, null, 2)); + core.info(`Graders: wrote results to ${RESULTS_PATH}`); + } catch (err) { + core.warning(`Graders: failed to write results: ${getErrorMessage(err)}`); + } + + // Step summary + core.summary.addHeading("Trace Graders", 3); + const tableResults = results.filter(r => r.status !== "unavailable"); + if (tableResults.length > 0) { + const rows = tableResults.map(r => { + const statusIcon = r.status === "pass" ? "✅" : r.status === "fail" ? "❌" : "⚠️"; + const val = r.value !== null ? String(Number(r.value.toFixed(4))) : "—"; + return [statusIcon, r.name, r.source, val, r.unit || "—"]; + }); + core.summary.addTable([ + [ + { data: "", header: true }, + { data: "Grader", header: true }, + { data: "Source", header: true }, + { data: "Value", header: true }, + { data: "Unit", header: true }, + ], + ...rows, + ]); + } + const errResults = results.filter(r => r.error); + if (errResults.length > 0) { + const errLines = errResults.map(r => `- **${r.id}**: ${r.error}`).join("\n"); + core.summary.addDetails("Grader Errors", errLines); + } + await core.summary.write({ overwrite: false }); + + core.info(`Graders: ${passed} passed, ${failed} failed, ${errorCount} errors`); +} + +module.exports = { + main, + preprocessTrace, + safeReadFile, + safeParseJsonl, + safeParseJson, + readFirstAvailable, + deepFreeze, + deepClone, + runGrader, + runBuiltinGrader, + runCustomGrader, + normalizeResult, + evaluateThreshold, + BUILTIN_GRADERS, + BUILTIN_META, + GRADER_VERSION, + IMPLEMENTATION_ID, + GRADERS_DIR, + MANIFEST_PATH, + RESULTS_PATH, + MAX_FILE_SIZE, + MAX_LINE_LENGTH, + SCRIPT_TIMEOUT_MS, + gradeToolSuccessRate, + gradeToolFailureCount, + gradeRetries, + gradeLoops, + gradeTrajectoryEfficiency, + gradeExecutionStepCount, + gradeExecutionDuration, + gradeContextGrowth, + gradeArtifactProduction, +}; diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs new file mode 100644 index 00000000000..bd710c0c8bb --- /dev/null +++ b/actions/setup/js/trace_graders.test.cjs @@ -0,0 +1,567 @@ +// @ts-check +/// + +const fs = require("fs"); +const path = require("path"); +const os = require("os"); + +const { + main, + preprocessTrace, + safeReadFile, + safeParseJsonl, + safeParseJson, + readFirstAvailable, + deepFreeze, + deepClone, + runGrader, + runBuiltinGrader, + runCustomGrader, + normalizeResult, + evaluateThreshold, + BUILTIN_GRADERS, + BUILTIN_META, + GRADER_VERSION, + IMPLEMENTATION_ID, + MANIFEST_PATH, + RESULTS_PATH, + MAX_FILE_SIZE, + MAX_LINE_LENGTH, + SCRIPT_TIMEOUT_MS, + gradeToolSuccessRate, + gradeToolFailureCount, + gradeRetries, + gradeLoops, + gradeTrajectoryEfficiency, + gradeExecutionStepCount, + gradeExecutionDuration, + gradeContextGrowth, + gradeArtifactProduction, +} = require("./trace_graders.cjs"); + +// --- Helper to create a minimal trace --- + +/** @returns {import("./trace_graders.cjs").PreprocessedTrace} */ +function makeTrace(overrides = {}) { + return { + tokenUsageEntries: [], + agentUsage: null, + mcpGatewayEntries: [], + agentOutput: null, + toolCalls: [], + gatewayRequests: [], + retryEvents: [], + errorEvents: [], + steps: [], + totalInputTokens: 0, + totalOutputTokens: 0, + totalDurationMs: 0, + totalRequests: 0, + files: [], + artifacts: [], + ...overrides, + }; +} + +describe("trace_graders", () => { + // --- safeParseJsonl --- + describe("safeParseJsonl", () => { + it("parses valid JSONL", () => { + const content = '{"a":1}\n{"b":2}\n'; + const result = safeParseJsonl(content); + expect(result).toEqual([{ a: 1 }, { b: 2 }]); + }); + + it("skips malformed lines", () => { + const content = '{"a":1}\nnot json\n{"b":2}\n'; + const result = safeParseJsonl(content); + expect(result).toEqual([{ a: 1 }, { b: 2 }]); + }); + + it("skips oversized lines", () => { + const longLine = '{"x":"' + "a".repeat(MAX_LINE_LENGTH + 10) + '"}'; + const content = '{"a":1}\n' + longLine + '\n{"b":2}\n'; + const result = safeParseJsonl(content); + expect(result).toEqual([{ a: 1 }, { b: 2 }]); + }); + + it("handles empty input", () => { + expect(safeParseJsonl("")).toEqual([]); + expect(safeParseJsonl("\n\n")).toEqual([]); + }); + }); + + // --- safeParseJson --- + describe("safeParseJson", () => { + it("parses valid JSON", () => { + expect(safeParseJson('{"a":1}')).toEqual({ a: 1 }); + }); + + it("returns null for invalid JSON", () => { + expect(safeParseJson("not json")).toBeNull(); + }); + + it("returns null for oversized content", () => { + const big = "a".repeat(MAX_FILE_SIZE + 1); + expect(safeParseJson(big)).toBeNull(); + }); + }); + + // --- deepFreeze --- + describe("deepFreeze", () => { + it("freezes objects deeply", () => { + const obj = { a: { b: { c: 1 } } }; + deepFreeze(obj); + expect(Object.isFrozen(obj)).toBe(true); + expect(Object.isFrozen(obj.a)).toBe(true); + expect(Object.isFrozen(obj.a.b)).toBe(true); + }); + + it("handles null/primitives", () => { + expect(deepFreeze(null)).toBeNull(); + expect(deepFreeze(42)).toBe(42); + }); + }); + + // --- Built-in graders --- + describe("gradeToolSuccessRate", () => { + it("returns 1 for no tool calls", () => { + expect(gradeToolSuccessRate(makeTrace())).toBe(1); + }); + + it("computes success rate", () => { + const trace = makeTrace({ + toolCalls: [{ name: "a", success: true }, { name: "b", success: false }, { name: "c", success: true }], + }); + expect(gradeToolSuccessRate(trace)).toBeCloseTo(2 / 3); + }); + + it("handles error field as failure indicator", () => { + const trace = makeTrace({ + toolCalls: [{ name: "a" }, { name: "b", error: "something failed" }], + }); + expect(gradeToolSuccessRate(trace)).toBe(0.5); + }); + }); + + describe("gradeToolFailureCount", () => { + it("returns 0 for no tool calls", () => { + expect(gradeToolFailureCount(makeTrace())).toBe(0); + }); + + it("counts failures", () => { + const trace = makeTrace({ + toolCalls: [{ name: "a", success: true }, { name: "b", success: false }, { name: "c", error: "err" }], + }); + expect(gradeToolFailureCount(trace)).toBe(2); + }); + }); + + describe("gradeRetries", () => { + it("returns 0 for no retries", () => { + expect(gradeRetries(makeTrace())).toBe(0); + }); + + it("counts retry events from preprocessed retryEvents", () => { + const trace = makeTrace({ + retryEvents: [{ event: "retry" }, { retry: true }, { message: "Retrying request" }], + }); + expect(gradeRetries(trace)).toBe(3); + }); + }); + + describe("gradeLoops", () => { + it("returns 0 for no loops", () => { + expect(gradeLoops(makeTrace())).toBe(0); + }); + + it("detects consecutive identical calls", () => { + const trace = makeTrace({ + toolCalls: [ + { name: "read", arguments: { path: "/a" } }, + { name: "read", arguments: { path: "/a" } }, + { name: "read", arguments: { path: "/a" } }, + { name: "write", arguments: { path: "/b" } }, + ], + }); + expect(gradeLoops(trace)).toBe(2); + }); + }); + + describe("gradeTrajectoryEfficiency", () => { + it("returns 1 for no tool calls", () => { + expect(gradeTrajectoryEfficiency(makeTrace())).toBe(1); + }); + + it("computes efficiency", () => { + const trace = makeTrace({ + toolCalls: [{ name: "read" }, { name: "write" }, { name: "read" }, { name: "read" }], + }); + expect(gradeTrajectoryEfficiency(trace)).toBe(0.5); + }); + }); + + describe("gradeExecutionStepCount", () => { + it("returns total LLM requests", () => { + expect(gradeExecutionStepCount(makeTrace({ totalRequests: 42 }))).toBe(42); + }); + }); + + describe("gradeExecutionDuration", () => { + it("returns total duration", () => { + expect(gradeExecutionDuration(makeTrace({ totalDurationMs: 12345 }))).toBe(12345); + }); + }); + + describe("gradeContextGrowth", () => { + it("returns 1 for fewer than 2 entries", () => { + expect(gradeContextGrowth(makeTrace())).toBe(1); + expect(gradeContextGrowth(makeTrace({ tokenUsageEntries: [{ input_tokens: 100, output_tokens: 50 }] }))).toBe(1); + }); + + it("computes growth ratio", () => { + const trace = makeTrace({ + tokenUsageEntries: [ + { input_tokens: 100, output_tokens: 50 }, + { input_tokens: 200, output_tokens: 100 }, + ], + totalInputTokens: 300, + totalOutputTokens: 150, + }); + expect(gradeContextGrowth(trace)).toBe(3); + }); + }); + + describe("gradeArtifactProduction", () => { + it("returns 0 for no agent output", () => { + expect(gradeArtifactProduction(makeTrace())).toBe(0); + }); + + it("counts artifacts array", () => { + const trace = makeTrace({ + artifacts: [{ type: "pr" }, { type: "issue" }], + }); + expect(gradeArtifactProduction(trace)).toBe(2); + }); + }); + + // --- normalizeResult --- + describe("normalizeResult", () => { + const meta = { name: "Test", unit: "count", direction: "lower_is_better", threshold: 5, source: "builtin" }; + + it("normalizes a number result with threshold pass", () => { + const r = normalizeResult("test", 3, meta); + expect(r.id).toBe("test"); + expect(r.value).toBe(3); + expect(r.passed).toBe(true); + expect(r.status).toBe("pass"); + expect(r.unit).toBe("count"); + expect(r.implementation.id).toBe(IMPLEMENTATION_ID); + }); + + it("normalizes a number result with threshold fail", () => { + const r = normalizeResult("test", 10, meta); + expect(r.passed).toBe(false); + expect(r.status).toBe("fail"); + }); + + it("handles object results from custom scripts", () => { + const r = normalizeResult("test", { value: 42, unit: "ms", severity: "warning", details: "too slow" }, { ...meta, source: "inline" }); + expect(r.value).toBe(42); + expect(r.unit).toBe("ms"); + expect(r.severity).toBe("warning"); + expect(r.details).toBe("too slow"); + }); + + it("handles null result as unavailable", () => { + const r = normalizeResult("test", null, meta); + expect(r.status).toBe("unavailable"); + }); + + it("handles non-finite value as error", () => { + const r = normalizeResult("test", NaN, meta); + expect(r.status).toBe("error"); + expect(r.error).toContain("non-finite"); + }); + + it("includes digest in implementation when provided", () => { + const r = normalizeResult("test", 1, { ...meta, source: "inline", digest: "abc123" }); + expect(r.implementation.digest).toBe("abc123"); + }); + }); + + // --- evaluateThreshold --- + describe("evaluateThreshold", () => { + it("passes when higher_is_better and value >= threshold", () => { + expect(evaluateThreshold(0.9, "higher_is_better", 0.8)).toBe(true); + expect(evaluateThreshold(0.7, "higher_is_better", 0.8)).toBe(false); + }); + + it("passes when lower_is_better and value <= threshold", () => { + expect(evaluateThreshold(3, "lower_is_better", 5)).toBe(true); + expect(evaluateThreshold(10, "lower_is_better", 5)).toBe(false); + }); + + it("returns null when no threshold", () => { + expect(evaluateThreshold(1, "higher_is_better", undefined)).toBeNull(); + }); + }); + + // --- runGrader --- + describe("runGrader", () => { + it("runs built-in grader", () => { + const trace = makeTrace({ totalRequests: 5 }); + const result = runGrader("execution-step-count", true, undefined, trace); + expect(result.value).toBe(5); + expect(result.error).toBeNull(); + }); + + it("runs inline script via vm sandbox", () => { + const trace = makeTrace({ toolCalls: [{ name: "a" }, { name: "b" }] }); + const result = runGrader("custom", false, "return { value: trace.toolCalls.length }", trace); + expect(result.value).toBe(2); + expect(result.error).toBeNull(); + }); + + it("supports legacy expression-style scripts", () => { + const trace = makeTrace({ toolCalls: [{ name: "a" }] }); + const result = runGrader("custom", false, "return trace.toolCalls.length", trace); + expect(result.value).toBe(1); + }); + + it("catches script errors", () => { + const result = runGrader("bad", false, "return undefinedVar.prop", makeTrace()); + expect(result.value).toBeNull(); + expect(result.error).toContain("runtime error"); + }); + + it("rejects non-numeric results from inline scripts", () => { + const result = runGrader("str", false, 'return "hello"', makeTrace()); + expect(result.value).toBeNull(); + expect(result.error).toContain("non-finite"); + }); + + it("rejects NaN from built-in graders", () => { + const result = runGrader("nan-test", false, "return NaN", makeTrace()); + expect(result.value).toBeNull(); + expect(result.error).toContain("non-finite"); + }); + + it("rejects Infinity", () => { + const result = runGrader("inf-test", false, "return Infinity", makeTrace()); + expect(result.value).toBeNull(); + expect(result.error).toContain("non-finite"); + }); + }); + + // --- runCustomGrader node:vm sandbox --- + describe("runCustomGrader sandbox", () => { + const meta = { name: "test", unit: "", direction: "", source: "inline" }; + + it("cannot access require", () => { + const result = runCustomGrader("test", "return typeof require", makeTrace(), meta); + expect(result.value).toBeNull(); // "undefined" is not a number + }); + + it("cannot access process", () => { + const result = runCustomGrader("test", "return typeof process", makeTrace(), meta); + expect(result.value).toBeNull(); // "undefined" is not a number + }); + + it("cannot access fetch", () => { + const result = runCustomGrader("test", "return typeof fetch", makeTrace(), meta); + expect(result.value).toBeNull(); + }); + + it("cannot construct Date", () => { + const result = runCustomGrader("test", "try { new Date(); return 0 } catch(e) { return 1 }", makeTrace(), meta); + // Date is not available in the sandbox + expect(result.value).toBe(1); + }); + + it("cannot use Math.random", () => { + const result = runCustomGrader("test", "return typeof Math.random", makeTrace(), meta); + expect(result.value).toBeNull(); // "undefined" is not a number + }); + + it("cannot mutate frozen trace", () => { + const trace = makeTrace({ toolCalls: [{ name: "a" }] }); + const result = runCustomGrader("test", "try { trace.toolCalls.push({name:'b'}); return 0 } catch(e) { return 1 }", trace, meta); + expect(result.value).toBe(1); + }); + + it("receives config parameter", () => { + const result = runCustomGrader("test", "return config.multiplier * 2", makeTrace(), { ...meta, config: { multiplier: 5 } }); + expect(result.value).toBe(10); + }); + + it("can use helpers", () => { + const result = runCustomGrader("test", "return helpers.clamp(10, 0, 5)", makeTrace(), meta); + expect(result.value).toBe(5); + }); + + it("supports multiline return object form", () => { + const script = ` + const count = trace.toolCalls.length; + return { + value: count, + unit: "count", + severity: count > 10 ? "warning" : "info" + } + `; + const trace = makeTrace({ toolCalls: [{ name: "a" }, { name: "b" }] }); + const result = runCustomGrader("test", script, trace, meta); + expect(result.value).toBe(2); + expect(result.unit).toBe("count"); + }); + + it("times out on infinite loops", () => { + const result = runCustomGrader("test", "while(true){} return 1", makeTrace(), meta); + expect(result.status).toBe("error"); + expect(result.error).toContain("runtime error"); + }); + }); + + // --- Hostile data --- + describe("hostile data handling", () => { + it("handles hostile strings in tool call names", () => { + const trace = makeTrace({ + toolCalls: [ + { name: '">', success: true }, + { name: "normal", success: true }, + ], + }); + expect(gradeToolSuccessRate(trace)).toBe(1); + expect(gradeToolFailureCount(trace)).toBe(0); + }); + + it("handles nested malicious JSON in JSONL", () => { + const hostile = '{"a": 1, "__proto__": {"polluted": true}}'; + const result = safeParseJsonl(hostile); + expect(result.length).toBe(1); + expect(result[0].a).toBe(1); + expect(Object.prototype).not.toHaveProperty("polluted"); + }); + + it("hostile script cannot escape sandbox", () => { + const meta = { name: "test", unit: "", direction: "", source: "inline" }; + // Attempt to access constructor chain + const result = runCustomGrader("test", "return this && this.constructor ? 0 : 1", makeTrace(), meta); + // Should not crash + expect(result.error === null || result.error !== null).toBe(true); + }); + + it("hostile script cannot use eval via string code gen", () => { + const meta = { name: "test", unit: "", direction: "", source: "inline" }; + // codeGeneration: {strings: false} prevents this + const result = runCustomGrader("test", "try { const f = new Function('return 1'); return 0 } catch(e) { return 1 }", makeTrace(), meta); + expect(result.value).toBe(1); // Should fail because code gen is disabled + }); + }); + + // --- Determinism --- + describe("determinism", () => { + it("produces identical results for identical input", () => { + const trace = makeTrace({ + toolCalls: [ + { name: "read", success: true, arguments: { path: "/a" } }, + { name: "write", success: false, arguments: { path: "/b" } }, + { name: "read", success: true, arguments: { path: "/a" } }, + ], + tokenUsageEntries: [ + { input_tokens: 100, output_tokens: 50, duration_ms: 1000 }, + { input_tokens: 200, output_tokens: 100, duration_ms: 2000 }, + ], + totalInputTokens: 300, + totalOutputTokens: 150, + totalDurationMs: 3000, + totalRequests: 2, + retryEvents: [{ event: "retry" }], + artifacts: [{ type: "pr" }], + steps: [ + { index: 0, inputTokens: 100, outputTokens: 50, durationMs: 1000, model: null }, + { index: 1, inputTokens: 200, outputTokens: 100, durationMs: 2000, model: null }, + ], + }); + + const results1 = Object.entries(BUILTIN_GRADERS).map(([id, fn]) => ({ + id, + value: fn(trace), + })); + const results2 = Object.entries(BUILTIN_GRADERS).map(([id, fn]) => ({ + id, + value: fn(trace), + })); + + expect(results1).toEqual(results2); + }); + + it("produces no timestamp in normalized output", () => { + const meta = { name: "Test", unit: "count", direction: "lower_is_better", source: "builtin" }; + const r = normalizeResult("test", 5, meta); + expect(r).not.toHaveProperty("timestamp"); + }); + }); + + // --- trace.steps and enriched fields --- + describe("preprocessTrace enrichment", () => { + it("extracts steps from token usage entries", () => { + // Mock safeReadFile to return test data - use the preprocessTrace's logic + const trace = makeTrace({ + tokenUsageEntries: [ + { input_tokens: 100, output_tokens: 50, duration_ms: 500, model: "gpt-4" }, + { input_tokens: 200, output_tokens: 100, duration_ms: 1000, model: "gpt-4" }, + ], + steps: [ + { index: 0, inputTokens: 100, outputTokens: 50, durationMs: 500, model: "gpt-4" }, + { index: 1, inputTokens: 200, outputTokens: 100, durationMs: 1000, model: "gpt-4" }, + ], + }); + expect(trace.steps.length).toBe(2); + expect(trace.steps[0].inputTokens).toBe(100); + expect(trace.steps[1].model).toBe("gpt-4"); + }); + + it("extracts retryEvents, errorEvents", () => { + const trace = makeTrace({ + retryEvents: [{ event: "retry" }], + errorEvents: [{ level: "error", message: "fail" }], + }); + expect(trace.retryEvents.length).toBe(1); + expect(trace.errorEvents.length).toBe(1); + }); + }); + + // --- All built-in graders are registered --- + describe("built-in grader registry", () => { + const expectedIds = ["tool-success-rate", "tool-failure-count", "retries", "loops", "trajectory-efficiency", "execution-step-count", "execution-duration", "context-growth", "artifact-production"]; + + it("has all expected built-in graders", () => { + for (const id of expectedIds) { + expect(BUILTIN_GRADERS).toHaveProperty(id); + expect(typeof BUILTIN_GRADERS[id]).toBe("function"); + } + }); + + it("has no unexpected graders", () => { + const ids = Object.keys(BUILTIN_GRADERS); + expect(ids.sort()).toEqual([...expectedIds].sort()); + }); + + it("all graders have metadata", () => { + for (const id of expectedIds) { + expect(BUILTIN_META).toHaveProperty(id); + expect(BUILTIN_META[id].unit).toBeDefined(); + expect(BUILTIN_META[id].direction).toBeDefined(); + } + }); + }); + + // --- GRADER_VERSION --- + describe("version", () => { + it("is a number", () => { + expect(typeof GRADER_VERSION).toBe("number"); + expect(GRADER_VERSION).toBe(1); + }); + }); +}); diff --git a/actions/setup/sh/prepare_threat_detection_files.sh b/actions/setup/sh/prepare_threat_detection_files.sh index 044fc804f5c..b676b6974cb 100755 --- a/actions/setup/sh/prepare_threat_detection_files.sh +++ b/actions/setup/sh/prepare_threat_detection_files.sh @@ -47,5 +47,14 @@ for artifact_pattern in aw-*.patch aw-*.bundle; do done done +# Copy grader manifest and results if present (deterministic trace graders) +GRADER_SOURCE_DIR="${SOURCE_DIR}/agent/graders" +if [ -d "${GRADER_SOURCE_DIR}" ]; then + GRADER_DETECTION_DIR="${DETECTION_DIR}/agent/graders" + mkdir -p "${GRADER_DETECTION_DIR}" + copy_optional_file "${GRADER_SOURCE_DIR}/grader_manifest.json" "${GRADER_DETECTION_DIR}/grader_manifest.json" + copy_optional_file "${GRADER_SOURCE_DIR}/grader_results.json" "${GRADER_DETECTION_DIR}/grader_results.json" +fi + echo "Prepared threat detection files:" ls -la "${DETECTION_DIR}" diff --git a/docs/src/content/docs/reference/trace-graders.md b/docs/src/content/docs/reference/trace-graders.md new file mode 100644 index 00000000000..2765f8a593e --- /dev/null +++ b/docs/src/content/docs/reference/trace-graders.md @@ -0,0 +1,63 @@ +--- +title: Trace Graders +description: Deterministic metrics computed from agent execution traces +--- + +Trace graders compute deterministic metrics from post-agent execution trace files (token usage, MCP gateway logs, agent output) without LLM calls or network access. Results are persisted in the agent artifact for downstream consumption by detection jobs and reporting tools. + +## Quick start + +```yaml +graders: {} +``` + +An empty map enables all built-in graders with default settings. Omitting the `graders` field entirely disables grading (no step is emitted). + +## Built-in graders + +| ID | Description | Value | +|---|---|---| +| `tool-success-rate` | Fraction of tool calls that succeeded | 0–1 | +| `tool-failure-count` | Number of failed tool calls | integer | +| `retries` | Count of retry events in MCP gateway logs | integer | +| `loops` | Consecutive identical tool calls (same name + args) | integer | +| `trajectory-efficiency` | Unique tool names / total tool calls | 0–1 | +| `execution-step-count` | Total LLM request count | integer | +| `execution-duration` | Total execution duration (ms) | integer | +| `context-growth` | Total tokens / first-request tokens | ≥1 | +| `artifact-production` | Count of outputs in agent_output.json | integer | + +## Selective configuration + +Disable a specific built-in: + +```yaml +graders: + loops: + enabled: false +``` + +## Custom inline graders + +Add a trusted inline JavaScript expression that receives the preprocessed `trace` object: + +```yaml +graders: + bash-calls: + script: "trace.toolCalls.filter(t => t.name === 'bash').length" +``` + +Custom scripts must be pure expressions (≤2 KB, no `require`, `import`, `fetch`, `eval`, or `process.exit`). + +## Output files + +| File | Description | +|---|---| +| `grader_manifest.json` | Which graders were configured and their enabled state | +| `grader_results.json` | Normalized metric values with trace summary | + +Both files are included in the unified `agent` artifact. + +## Execution + +The graders step runs as an `if: always()` post-agent step in the existing agent job, after log parsing and before the unified artifact upload. It uses a single preprocessing pass over trace files shared by all graders. diff --git a/pkg/constants/job_constants.go b/pkg/constants/job_constants.go index 2fd11f87b09..a373f9f7ac0 100644 --- a/pkg/constants/job_constants.go +++ b/pkg/constants/job_constants.go @@ -130,6 +130,20 @@ const SafeOutputsFilename = "safeoutputs.jsonl" // consume structured token data without parsing the step summary or GITHUB_OUTPUT. const TokenUsageFilename = "agent_usage.json" +// GraderManifestFilename is the filename of the grader manifest JSON written to /tmp/gh-aw/agent/graders/ +// by trace_graders.cjs. Lists which graders were configured and their enabled state. +const GraderManifestFilename = "grader_manifest.json" + +// GraderResultsFilename is the filename of the normalized grader results JSON written to /tmp/gh-aw/agent/graders/ +// by trace_graders.cjs. Contains deterministic metric values computed from trace files. +const GraderResultsFilename = "grader_results.json" + +// GradersDir is the subdirectory under TmpGhAwAgentDir where grader output files are written. +const GradersDir = TmpGhAwDir + "/agent/graders" + +// GradersDirSlash is GradersDir with a trailing slash. +const GradersDirSlash = GradersDir + "/" + // GithubRateLimitsFilename is the filename of the GitHub API rate-limit log written to /tmp/gh-aw/. // Each line is a JSON object recording the x-ratelimit-* headers (or rate-limit API snapshot) // captured during github.rest API calls, enabling post-run analysis of rate-limit consumption. diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 330bf12dcfb..72e7bba742f 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -564,6 +564,13 @@ func (c *Compiler) extractAdditionalConfigurations( return fmt.Errorf("invalid experiments configuration: %w", err) } + // Extract deterministic trace graders configuration. + gradersConfig, err := c.parseGradersFromFrontmatter(frontmatter) + if err != nil { + return fmt.Errorf("invalid graders configuration: %w", err) + } + workflowData.Graders = gradersConfig + return nil } diff --git a/pkg/workflow/compiler_yaml_artifacts.go b/pkg/workflow/compiler_yaml_artifacts.go index 4c8193a39e1..2f91bb04eb1 100644 --- a/pkg/workflow/compiler_yaml_artifacts.go +++ b/pkg/workflow/compiler_yaml_artifacts.go @@ -75,6 +75,15 @@ func (c *Compiler) generateAgentOutputFallbackUpload(yaml *strings.Builder, data constants.TmpGhAwDirSlash + constants.SafeOutputsFilename, } + // Include grader manifest/results in the fallback so detection and downstream + // jobs have reliable access even when the large unified artifact times out. + if data.Graders != nil && data.Graders.HasGraders() { + paths = append(paths, + constants.GradersDirSlash+constants.GraderManifestFilename, + constants.GradersDirSlash+constants.GraderResultsFilename, + ) + } + c.stepOrderTracker.RecordArtifactUpload("Upload agent output fallback artifact", paths) yaml.WriteString(" # Small dedicated copy of the agent output so safe-output processing\n") diff --git a/pkg/workflow/compiler_yaml_graders.go b/pkg/workflow/compiler_yaml_graders.go new file mode 100644 index 00000000000..aab0ee9ccea --- /dev/null +++ b/pkg/workflow/compiler_yaml_graders.go @@ -0,0 +1,227 @@ +package workflow + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/logger" +) + +var compilerYamlGradersLog = logger.New("workflow:compiler_yaml_graders") + +// generateGradersStep emits an always() post-agent step that runs deterministic +// trace graders. The step executes after secret redaction / summary steps and before +// the unified artifact upload so results are included in the agent artifact. +// +// The step is only emitted when graders are configured (graders: in frontmatter). +func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData) { + if data.Graders == nil || !data.Graders.HasGraders() { + return + } + + compilerYamlGradersLog.Printf("Generating graders step with %d enabled graders", len(data.Graders.EnabledGraderIDs())) + + // Build the manifest JSON that the JS runtime will consume. + manifest := buildGraderManifest(data.Graders) + manifestJSON, err := json.Marshal(manifest) + if err != nil { + compilerYamlGradersLog.Printf("Failed to marshal grader manifest: %v", err) + return + } + + // Build execution spec (scripts) separately, base64 encoded for safety. + execSpec := buildGraderExecSpec(data.Graders) + execJSON, err := json.Marshal(execSpec) + if err != nil { + compilerYamlGradersLog.Printf("Failed to marshal grader exec spec: %v", err) + return + } + execB64 := base64.StdEncoding.EncodeToString(execJSON) + + // Escape single quotes for embedding in the YAML script block + escapedManifest := strings.ReplaceAll(string(manifestJSON), "'", "\\'") + + yaml.WriteString(" - name: Run trace graders\n") + yaml.WriteString(" if: always()\n") + yaml.WriteString(" continue-on-error: true\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { main } = require('" + SetupActionDestination + "/trace_graders.cjs');\n") + fmt.Fprintf(yaml, " await main('%s', '%s');\n", escapedManifest, execB64) + + compilerYamlGradersLog.Print("Generated trace graders step") +} + +// graderManifestEntry represents a single grader in the serialized manifest. +// The manifest is an object {version:1, graders:[...]} for stable schema. +type graderManifestEntry struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Source string `json:"source"` // "builtin" or "inline" + Enabled bool `json:"enabled"` + Unit string `json:"unit,omitempty"` + Direction string `json:"direction,omitempty"` + Threshold *float64 `json:"threshold,omitempty"` + Max *float64 `json:"max,omitempty"` + Min *float64 `json:"min,omitempty"` + Digest string `json:"digest,omitempty"` // SHA-256 of inline script + Config map[string]any `json:"config,omitempty"` +} + +// graderManifest is the top-level manifest object written to disk. +type graderManifest struct { + Version int `json:"version"` + Graders []graderManifestEntry `json:"graders"` +} + +// graderExecEntry carries the script body for a custom grader, keyed by ID. +type graderExecEntry struct { + ID string `json:"id"` + Script string `json:"script"` +} + +// buildGraderManifest constructs the manifest for the JS runtime. +func buildGraderManifest(cfg *GradersConfig) *graderManifest { + if cfg == nil { + return &graderManifest{Version: 1} + } + + builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + for _, id := range BuiltinGraderIDs { + builtinSet[id] = true + } + + ids := cfg.EnabledGraderIDs() + // Also include disabled graders so the manifest records them + var disabledIDs []string + for id, g := range cfg.Graders { + if g.Enabled != nil && !*g.Enabled { + disabledIDs = append(disabledIDs, id) + } + } + sortStrings(disabledIDs) + + entries := make([]graderManifestEntry, 0, len(ids)+len(disabledIDs)) + + addEntry := func(id string, enabled bool) { + g := cfg.Graders[id] + source := "builtin" + if !builtinSet[id] { + source = "inline" + } + name := g.Name + if name == "" { + name = id + } + entries = append(entries, graderManifestEntry{ + ID: id, + Name: name, + Description: g.Description, + Source: source, + Enabled: enabled, + Unit: g.Unit, + Direction: g.Direction, + Threshold: g.Threshold, + Max: g.Max, + Min: g.Min, + Digest: g.ScriptDigest(), + Config: g.Config, + }) + } + + for _, id := range ids { + addEntry(id, true) + } + for _, id := range disabledIDs { + addEntry(id, false) + } + + return &graderManifest{Version: 1, Graders: entries} +} + +// buildGraderExecSpec builds the execution spec: an array of {id, script} +// entries for custom graders only. This is base64-encoded to avoid JS/YAML injection. +func buildGraderExecSpec(cfg *GradersConfig) []graderExecEntry { + if cfg == nil { + return nil + } + builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + for _, id := range BuiltinGraderIDs { + builtinSet[id] = true + } + + var specs []graderExecEntry + for _, id := range cfg.EnabledGraderIDs() { + g := cfg.Graders[id] + if !builtinSet[id] && g.Script != "" { + specs = append(specs, graderExecEntry{ID: id, Script: g.Script}) + } + } + return specs +} + +// sortStrings sorts a string slice in place. +func sortStrings(s []string) { + for i := 0; i < len(s); i++ { + for j := i + 1; j < len(s); j++ { + if s[j] < s[i] { + s[i], s[j] = s[j], s[i] + } + } + } +} + +// generateGraderRedactionStep emits a lightweight redaction pass that scans grader +// output files for leaked secrets. Custom grader scripts can evaluate trace data +// that may contain credential-bearing strings. This step runs after the graders step +// and reuses the existing redact_secrets.cjs infrastructure. +func (c *Compiler) generateGraderRedactionStep(yaml *strings.Builder, yamlContent string, data *WorkflowData) { + if data.Graders == nil || !data.Graders.HasGraders() { + return + } + if !data.Graders.HasCustomScripts() { + return + } + + secretReferences := CollectSecretReferences(yamlContent) + c.stepOrderTracker.RecordSecretRedaction("Redact grader outputs") + + yaml.WriteString(" - name: Redact grader outputs\n") + yaml.WriteString(" if: always()\n") + yaml.WriteString(" continue-on-error: true\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { redactFilesInDir } = require('" + SetupActionDestination + "/redact_secrets.cjs');\n") + fmt.Fprintf(yaml, " await redactFilesInDir('%s');\n", constants.GradersDir) + + if len(secretReferences) > 0 { + yaml.WriteString(" env:\n") + escapedRefs := make([]string, len(secretReferences)) + for i, ref := range secretReferences { + escapedRefs[i] = escapeSingleQuoteBackslash(ref) + } + fmt.Fprintf(yaml, " GH_AW_SECRET_NAMES: '%s'\n", strings.Join(escapedRefs, ",")) + for _, secretName := range secretReferences { + escapedSecretName := escapeSingleQuoteBackslash(secretName) + fmt.Fprintf(yaml, " SECRET_%s: ${{ secrets.%s }}\n", escapedSecretName, secretName) + } + } +} + +// collectGraderArtifactPaths returns artifact paths for grader output files. +func collectGraderArtifactPaths() []string { + return []string{ + constants.GradersDirSlash + constants.GraderManifestFilename, + constants.GradersDirSlash + constants.GraderResultsFilename, + } +} diff --git a/pkg/workflow/compiler_yaml_post_agent.go b/pkg/workflow/compiler_yaml_post_agent.go index 442257e01ee..64ee82e95e5 100644 --- a/pkg/workflow/compiler_yaml_post_agent.go +++ b/pkg/workflow/compiler_yaml_post_agent.go @@ -60,6 +60,11 @@ func (c *Compiler) collectArtifactPaths(data *WorkflowData, engine CodingAgentEn paths = append(paths, constants.TmpGhAwDirSlash+constants.OtlpExportErrorsFilename) } + // Collect grader manifest and results when graders are configured. + if data.Graders != nil && data.Graders.HasGraders() { + paths = append(paths, collectGraderArtifactPaths()...) + } + // Collect safe outputs and agent output paths for the unified artifact. // These were previously uploaded as separate safe-output and agent-output artifacts. if data.SafeOutputs != nil { @@ -192,6 +197,14 @@ func (c *Compiler) generatePostAgentCollectionAndUpload(yaml *strings.Builder, d // Emit all GITHUB_STEP_SUMMARY log-parsing steps. c.generateSummarySteps(yaml, data, engine) + // Run deterministic trace graders after trace data is available. + c.generateGradersStep(yaml, data) + + // Re-scan grader output files for leaked secrets when custom grader scripts + // are present. Custom scripts evaluate trace data that may contain + // credential-bearing strings written after the initial workspace redaction. + c.generateGraderRedactionStep(yaml, yaml.String(), data) + // Write a minimal agent_output.json placeholder when the engine fails before // producing any safe outputs, so downstream safe_outputs and conclusion jobs // receive a valid (empty) JSON file instead of an ENOENT error. diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go index fe9cb1b254f..0c92ec2b86c 100644 --- a/pkg/workflow/frontmatter_types.go +++ b/pkg/workflow/frontmatter_types.go @@ -441,6 +441,11 @@ type FrontmatterConfig struct { // engine-config / runs-on overrides. Evals any `json:"evals,omitempty"` + // Graders configures deterministic trace graders that compute metrics from + // post-agent trace files. Can be {} for zero-config (all built-ins) or a map + // of grader IDs with optional enabled/script overrides. + Graders any `json:"graders,omitempty"` + // ExcludedEnv lists additional environment variable names that must be excluded from // the agent container via AWF's --exclude-env flag. Use this when an env var is set // from a source that the compiler cannot automatically detect as credential-bearing diff --git a/pkg/workflow/graders_config.go b/pkg/workflow/graders_config.go new file mode 100644 index 00000000000..12924c038e6 --- /dev/null +++ b/pkg/workflow/graders_config.go @@ -0,0 +1,414 @@ +// Package workflow - Deterministic trace graders configuration types and parser. +package workflow + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/github/gh-aw/pkg/logger" +) + +var gradersConfigLog = logger.New("workflow:graders_config") + +// BuiltinGraderMeta describes a built-in deterministic grader with its default metadata. +type BuiltinGraderMeta struct { + ID string + Name string + Description string + Unit string + Direction string // "higher_is_better" | "lower_is_better" + Threshold *float64 + Max *float64 + Min *float64 +} + +func ptrFloat(f float64) *float64 { return &f } + +// BuiltinGraderRegistry is the ordered list of all built-in grader definitions. +var BuiltinGraderRegistry = []BuiltinGraderMeta{ + {ID: "tool-success-rate", Name: "Tool Success Rate", Description: "Fraction of tool calls that succeeded", Unit: "ratio", Direction: "higher_is_better", Threshold: ptrFloat(0.8), Min: ptrFloat(0), Max: ptrFloat(1)}, + {ID: "tool-failure-count", Name: "Tool Failure Count", Description: "Number of tool calls that failed", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(5)}, + {ID: "retries", Name: "Retries", Description: "Number of retry events detected in gateway logs", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(10)}, + {ID: "loops", Name: "Loops", Description: "Consecutive identical tool calls (same name and arguments)", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(3)}, + {ID: "trajectory-efficiency", Name: "Trajectory Efficiency", Description: "Ratio of unique tool names to total tool calls (higher = more diverse usage)", Unit: "ratio", Direction: "higher_is_better", Min: ptrFloat(0), Max: ptrFloat(1)}, + {ID: "execution-step-count", Name: "Execution Step Count", Description: "Total LLM request count", Unit: "count", Direction: "lower_is_better"}, + {ID: "execution-duration", Name: "Execution Duration", Description: "Total execution duration", Unit: "ms", Direction: "lower_is_better"}, + {ID: "context-growth", Name: "Context Growth", Description: "Ratio of total tokens to first-request tokens", Unit: "factor", Direction: "lower_is_better"}, + {ID: "artifact-production", Name: "Artifact Production", Description: "Count of outputs/artifacts produced by the agent", Unit: "count", Direction: "higher_is_better"}, +} + +// BuiltinGraderIDs is the ordered list of built-in grader IDs (derived from registry). +var BuiltinGraderIDs = func() []string { + ids := make([]string, len(BuiltinGraderRegistry)) + for i, m := range BuiltinGraderRegistry { + ids[i] = m.ID + } + return ids +}() + +// builtinGraderMetaByID is a lookup map for BuiltinGraderRegistry. +var builtinGraderMetaByID = func() map[string]*BuiltinGraderMeta { + m := make(map[string]*BuiltinGraderMeta, len(BuiltinGraderRegistry)) + for i := range BuiltinGraderRegistry { + m[BuiltinGraderRegistry[i].ID] = &BuiltinGraderRegistry[i] + } + return m +}() + +// GraderDefinition represents a single grader entry in the graders map. +type GraderDefinition struct { + ID string // grader identifier (must be unique) + Enabled *bool // explicit enable/disable; nil means use default (true for built-ins) + Name string // human-readable name (defaults from registry for built-ins) + Description string // description of the metric + Unit string // e.g. "ratio", "count", "ms", "factor" + Direction string // "higher_is_better" or "lower_is_better" + Threshold *float64 // quality threshold (pass/fail boundary) + Max *float64 // theoretical maximum + Min *float64 // theoretical minimum + Script string // inline JS body for trusted custom graders (built-ins leave empty) + Config map[string]any // arbitrary config passed to grader at runtime +} + +// ScriptDigest returns the SHA-256 hex digest of the script, or "" if no script. +func (g *GraderDefinition) ScriptDigest() string { + if g.Script == "" { + return "" + } + h := sha256.Sum256([]byte(g.Script)) + return hex.EncodeToString(h[:]) +} + +// GradersConfig holds the configuration for deterministic trace graders declared +// in workflow frontmatter. Graders run as an always() post-agent step in the agent job. +type GradersConfig struct { + // Graders is the map of grader ID to definition. + Graders map[string]*GraderDefinition +} + +// HasGraders returns true when the config contains at least one enabled grader. +func (gc *GradersConfig) HasGraders() bool { + if gc == nil { + return false + } + for _, g := range gc.Graders { + if g.Enabled == nil || *g.Enabled { + return true + } + } + return false +} + +// HasCustomScripts returns true if any enabled grader has a custom script. +func (gc *GradersConfig) HasCustomScripts() bool { + if gc == nil { + return false + } + for _, g := range gc.Graders { + if (g.Enabled == nil || *g.Enabled) && g.Script != "" { + return true + } + } + return false +} + +// EnabledGraderIDs returns the sorted list of enabled grader IDs. +func (gc *GradersConfig) EnabledGraderIDs() []string { + if gc == nil { + return nil + } + enabledSet := make(map[string]bool) + for id, g := range gc.Graders { + if g.Enabled == nil || *g.Enabled { + enabledSet[id] = true + } + } + // Stable order: built-ins first in canonical order, then custom sorted + var result []string + builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + for _, bid := range BuiltinGraderIDs { + builtinSet[bid] = true + if enabledSet[bid] { + result = append(result, bid) + } + } + var custom []string + for id := range enabledSet { + if !builtinSet[id] { + custom = append(custom, id) + } + } + sort.Strings(custom) + result = append(result, custom...) + return result +} + +// graderIDPattern validates grader IDs: lowercase alphanumeric + hyphens, 1-64 chars. +var graderIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`) + +// parseGradersFromFrontmatter extracts and validates the graders configuration from the +// raw frontmatter map. Returns nil when the graders field is absent. +// +// Supported forms: +// +// # Zero-config: all built-ins enabled +// graders: {} +// +// # Selective disable +// graders: +// loops: +// enabled: false +// +// # Built-in with threshold override +// graders: +// tool-success-rate: +// threshold: 0.95 +// +// # Custom inline grader +// graders: +// my-metric: +// script: | +// return { value: trace.toolCalls.length } +// unit: count +// direction: lower_is_better +func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*GradersConfig, error) { + raw, exists := frontmatter["graders"] + if !exists || raw == nil { + return nil, nil + } + + cfg := &GradersConfig{ + Graders: make(map[string]*GraderDefinition), + } + + m, ok := raw.(map[string]any) + if !ok { + return nil, errors.New("graders must be a map of grader IDs to configuration objects (or {} for all built-in defaults). Example:\ngraders:\n tool-success-rate:\n enabled: true") + } + + builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + for _, id := range BuiltinGraderIDs { + builtinSet[id] = true + } + + // If empty map {}, populate all built-ins with defaults + if len(m) == 0 { + for _, id := range BuiltinGraderIDs { + meta := builtinGraderMetaByID[id] + cfg.Graders[id] = builtinDefFromMeta(meta) + } + gradersConfigLog.Printf("Parsed graders config: zero-config with %d built-in graders", len(BuiltinGraderIDs)) + return cfg, nil + } + + // Parse explicit entries + for id, entryRaw := range m { + id = strings.TrimSpace(id) + if !graderIDPattern.MatchString(id) { + return nil, fmt.Errorf("graders has invalid id %q: must match %s. Example:\ngraders:\n my-metric:\n script: \"return { value: trace.toolCalls.length }\"", id, graderIDPattern.String()) + } + + def := &GraderDefinition{ID: id} + + // Apply built-in defaults if this is a built-in + if meta, ok := builtinGraderMetaByID[id]; ok { + def = builtinDefFromMeta(meta) + } + + if entryRaw == nil { + cfg.Graders[id] = def + continue + } + + entry, ok := entryRaw.(map[string]any) + if !ok { + return nil, fmt.Errorf("graders.%s must be a map or null, got %T. Example:\ngraders:\n %s:\n enabled: true", id, entryRaw, id) + } + + if err := parseGraderEntryFields(def, entry, id, builtinSet[id]); err != nil { + return nil, err + } + + // Custom graders must have a script + if !builtinSet[id] && def.Script == "" && (def.Enabled == nil || *def.Enabled) { + return nil, fmt.Errorf("graders.%s is not a built-in grader and requires a 'script' field. Built-in graders: %s", id, strings.Join(BuiltinGraderIDs, ", ")) + } + + cfg.Graders[id] = def + } + + // Add missing built-ins as defaults when at least one built-in is explicitly listed + hasAnyBuiltin := false + for id := range cfg.Graders { + if builtinSet[id] { + hasAnyBuiltin = true + break + } + } + if hasAnyBuiltin { + for _, id := range BuiltinGraderIDs { + if _, exists := cfg.Graders[id]; !exists { + meta := builtinGraderMetaByID[id] + cfg.Graders[id] = builtinDefFromMeta(meta) + } + } + } + + if err := validateGraders(cfg); err != nil { + return nil, err + } + + enabledCount := len(cfg.EnabledGraderIDs()) + gradersConfigLog.Printf("Parsed %d grader definitions (%d enabled)", len(cfg.Graders), enabledCount) + return cfg, nil +} + +// builtinDefFromMeta creates a GraderDefinition from built-in metadata. +func builtinDefFromMeta(meta *BuiltinGraderMeta) *GraderDefinition { + def := &GraderDefinition{ + ID: meta.ID, + Name: meta.Name, + Description: meta.Description, + Unit: meta.Unit, + Direction: meta.Direction, + } + if meta.Threshold != nil { + v := *meta.Threshold + def.Threshold = &v + } + if meta.Max != nil { + v := *meta.Max + def.Max = &v + } + if meta.Min != nil { + v := *meta.Min + def.Min = &v + } + return def +} + +// parseGraderEntryFields parses individual fields from a grader entry map into the definition. +func parseGraderEntryFields(def *GraderDefinition, entry map[string]any, id string, isBuiltin bool) error { + if v, ok := entry["enabled"]; ok { + b, ok := v.(bool) + if !ok { + return fmt.Errorf("graders.%s.enabled must be a boolean, got %T", id, v) + } + def.Enabled = &b + } + + if v, ok := entry["name"]; ok { + s, ok := v.(string) + if !ok { + return fmt.Errorf("graders.%s.name must be a string, got %T", id, v) + } + def.Name = s + } + if v, ok := entry["description"]; ok { + s, ok := v.(string) + if !ok { + return fmt.Errorf("graders.%s.description must be a string, got %T", id, v) + } + def.Description = s + } + if v, ok := entry["unit"]; ok { + s, ok := v.(string) + if !ok { + return fmt.Errorf("graders.%s.unit must be a string, got %T", id, v) + } + def.Unit = s + } + if v, ok := entry["direction"]; ok { + s, ok := v.(string) + if !ok { + return fmt.Errorf("graders.%s.direction must be a string, got %T", id, v) + } + if s != "higher_is_better" && s != "lower_is_better" { + return fmt.Errorf("graders.%s.direction must be 'higher_is_better' or 'lower_is_better', got %q", id, s) + } + def.Direction = s + } + if err := parseOptionalFloat(entry, "threshold", id, &def.Threshold); err != nil { + return err + } + if err := parseOptionalFloat(entry, "max", id, &def.Max); err != nil { + return err + } + if err := parseOptionalFloat(entry, "min", id, &def.Min); err != nil { + return err + } + + if v, ok := entry["config"]; ok { + m, ok := v.(map[string]any) + if !ok { + return fmt.Errorf("graders.%s.config must be an object, got %T", id, v) + } + def.Config = m + } + + if scriptRaw, ok := entry["script"]; ok { + s, ok := scriptRaw.(string) + if !ok { + return fmt.Errorf("graders.%s.script must be a string, got %T", id, scriptRaw) + } + s = strings.TrimSpace(s) + if s == "" { + return fmt.Errorf("graders.%s.script must be non-empty when specified", id) + } + if isBuiltin { + return fmt.Errorf("graders.%s is a built-in grader and cannot have a custom script", id) + } + if len(s) > 4096 { + return fmt.Errorf("graders.%s.script exceeds maximum length of 4096 characters (%d)", id, len(s)) + } + forbiddenPatterns := []string{"require(", "import(", "import ", "fetch(", "eval(", "process.exit", "child_process", "execSync", "spawnSync", "Function("} + for _, p := range forbiddenPatterns { + if strings.Contains(s, p) { + return fmt.Errorf("graders.%s.script contains forbidden pattern %q — inline grader scripts must be pure functions without side effects", id, p) + } + } + def.Script = s + } + + return nil +} + +// parseOptionalFloat parses an optional float64 field from a map. +func parseOptionalFloat(m map[string]any, key string, graderID string, target **float64) error { + v, ok := m[key] + if !ok { + return nil + } + switch n := v.(type) { + case float64: + *target = &n + case int: + f := float64(n) + *target = &f + default: + return fmt.Errorf("graders.%s.%s must be a number, got %T", graderID, key, v) + } + return nil +} + +// validateGraders checks invariants after parsing. +func validateGraders(cfg *GradersConfig) error { + if cfg == nil { + return nil + } + if !cfg.HasGraders() { + return errors.New("graders configuration has no enabled graders. Remove the graders field to disable grading, or set enabled: true on at least one grader") + } + return nil +} + +// ParseGradersFromFrontmatter is a public standalone convenience wrapper. +func ParseGradersFromFrontmatter(frontmatter map[string]any) (*GradersConfig, error) { + var c Compiler + return c.parseGradersFromFrontmatter(frontmatter) +} diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go new file mode 100644 index 00000000000..906589fe31b --- /dev/null +++ b/pkg/workflow/graders_config_test.go @@ -0,0 +1,381 @@ +package workflow + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestParseGradersFromFrontmatter_Absent verifies nil return when graders absent. +func TestParseGradersFromFrontmatter_Absent(t *testing.T) { + var c Compiler + cfg, err := c.parseGradersFromFrontmatter(map[string]any{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg != nil { + t.Fatal("expected nil config when graders absent") + } +} + +// TestParseGradersFromFrontmatter_Nil verifies nil return when graders is nil. +func TestParseGradersFromFrontmatter_Nil(t *testing.T) { + var c Compiler + cfg, err := c.parseGradersFromFrontmatter(map[string]any{"graders": nil}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg != nil { + t.Fatal("expected nil config when graders is nil") + } +} + +// TestParseGradersFromFrontmatter_ZeroConfig verifies {} enables all built-ins. +func TestParseGradersFromFrontmatter_ZeroConfig(t *testing.T) { + var c Compiler + cfg, err := c.parseGradersFromFrontmatter(map[string]any{"graders": map[string]any{}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if !cfg.HasGraders() { + t.Fatal("expected HasGraders to be true") + } + ids := cfg.EnabledGraderIDs() + if len(ids) != len(BuiltinGraderIDs) { + t.Fatalf("expected %d enabled graders, got %d", len(BuiltinGraderIDs), len(ids)) + } +} + +// TestParseGradersFromFrontmatter_DisableOne verifies selective disable. +func TestParseGradersFromFrontmatter_DisableOne(t *testing.T) { + var c Compiler + cfg, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "loops": map[string]any{"enabled": false}, + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + ids := cfg.EnabledGraderIDs() + for _, id := range ids { + if id == "loops" { + t.Fatal("loops should be disabled") + } + } + // Should have all built-ins minus loops + if len(ids) != len(BuiltinGraderIDs)-1 { + t.Fatalf("expected %d enabled graders, got %d", len(BuiltinGraderIDs)-1, len(ids)) + } +} + +// TestParseGradersFromFrontmatter_CustomGrader verifies custom grader with script. +func TestParseGradersFromFrontmatter_CustomGrader(t *testing.T) { + var c Compiler + cfg, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "my-metric": map[string]any{ + "script": "trace.toolCalls.length", + }, + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + g, ok := cfg.Graders["my-metric"] + if !ok { + t.Fatal("expected my-metric grader") + } + if g.Script != "trace.toolCalls.length" { + t.Fatalf("unexpected script: %s", g.Script) + } +} + +// TestParseGradersFromFrontmatter_InvalidType verifies error for wrong type. +func TestParseGradersFromFrontmatter_InvalidType(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{"graders": "invalid"}) + if err == nil { + t.Fatal("expected error for string graders value") + } +} + +// TestParseGradersFromFrontmatter_ForbiddenScript verifies forbidden patterns in scripts. +func TestParseGradersFromFrontmatter_ForbiddenScript(t *testing.T) { + var c Compiler + forbidden := []string{ + "require('fs')", + "import('os')", + "fetch('http://evil.com')", + "eval('bad')", + "process.exit(1)", + } + for _, script := range forbidden { + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "bad-grader": map[string]any{"script": script}, + }, + }) + if err == nil { + t.Fatalf("expected error for forbidden script: %s", script) + } + if !strings.Contains(err.Error(), "forbidden pattern") { + t.Fatalf("expected forbidden pattern error, got: %v", err) + } + } +} + +// TestParseGradersFromFrontmatter_BuiltinScriptRejected verifies built-in cannot have script. +func TestParseGradersFromFrontmatter_BuiltinScriptRejected(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "retries": map[string]any{"script": "1 + 1"}, + }, + }) + if err == nil { + t.Fatal("expected error for built-in with script") + } +} + +// TestParseGradersFromFrontmatter_CustomWithoutScript verifies custom requires script. +func TestParseGradersFromFrontmatter_CustomWithoutScript(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "no-script": map[string]any{}, + }, + }) + if err == nil { + t.Fatal("expected error for custom grader without script") + } +} + +// TestParseGradersFromFrontmatter_InvalidID verifies ID validation. +func TestParseGradersFromFrontmatter_InvalidID(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "UPPER_CASE": map[string]any{"script": "1"}, + }, + }) + if err == nil { + t.Fatal("expected error for invalid ID") + } +} + +// TestParseGradersFromFrontmatter_AllDisabledError verifies error when all disabled. +func TestParseGradersFromFrontmatter_AllDisabledError(t *testing.T) { + var c Compiler + graders := map[string]any{} + for _, id := range BuiltinGraderIDs { + graders[id] = map[string]any{"enabled": false} + } + _, err := c.parseGradersFromFrontmatter(map[string]any{"graders": graders}) + if err == nil { + t.Fatal("expected error when all graders disabled") + } +} + +// TestGradersConfig_EnabledGraderIDs_Order verifies stable ordering. +func TestGradersConfig_EnabledGraderIDs_Order(t *testing.T) { + cfg := &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "zebra-metric": {ID: "zebra-metric", Script: "1"}, + "alpha-metric": {ID: "alpha-metric", Script: "1"}, + "retries": {ID: "retries"}, + "tool-success-rate": {ID: "tool-success-rate"}, + }, + } + ids := cfg.EnabledGraderIDs() + // Built-ins first in canonical order, then custom alphabetically + if ids[0] != "tool-success-rate" { + t.Fatalf("expected tool-success-rate first, got %s", ids[0]) + } + if ids[1] != "retries" { + t.Fatalf("expected retries second, got %s", ids[1]) + } + if ids[2] != "alpha-metric" { + t.Fatalf("expected alpha-metric third, got %s", ids[2]) + } + if ids[3] != "zebra-metric" { + t.Fatalf("expected zebra-metric fourth, got %s", ids[3]) + } +} + +// TestBuildGraderManifest verifies manifest serialization. +func TestBuildGraderManifest(t *testing.T) { + enabled := true + disabled := false + cfg := &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "tool-success-rate": {ID: "tool-success-rate", Enabled: &enabled}, + "retries": {ID: "retries", Enabled: &disabled}, + "my-custom": {ID: "my-custom", Script: "trace.toolCalls.length"}, + }, + } + entries := buildGraderManifest(cfg) + if len(entries) != 3 { + t.Fatalf("expected 3 entries, got %d", len(entries)) + } + + // Verify JSON serialization round-trips + data, err := json.Marshal(entries) + if err != nil { + t.Fatalf("json marshal error: %v", err) + } + var decoded []graderManifestEntry + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("json unmarshal error: %v", err) + } + if len(decoded) != 3 { + t.Fatalf("expected 3 decoded entries, got %d", len(decoded)) + } +} + +// TestGenerateGradersStep_Absent verifies no step when graders nil. +func TestGenerateGradersStep_Absent(t *testing.T) { + c := &Compiler{} + var yaml strings.Builder + data := &WorkflowData{Graders: nil} + c.generateGradersStep(&yaml, data) + if yaml.Len() != 0 { + t.Fatal("expected no output when graders nil") + } +} + +// TestGenerateGradersStep_Present verifies step is emitted. +func TestGenerateGradersStep_Present(t *testing.T) { + c := &Compiler{} + initActionPinCacheForTest(c) + var yaml strings.Builder + data := &WorkflowData{ + Graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "retries": {ID: "retries"}, + }, + }, + } + c.generateGradersStep(&yaml, data) + output := yaml.String() + if !strings.Contains(output, "Run trace graders") { + t.Fatal("expected step name 'Run trace graders'") + } + if !strings.Contains(output, "if: always()") { + t.Fatal("expected always() condition") + } + if !strings.Contains(output, "trace_graders.cjs") { + t.Fatal("expected trace_graders.cjs require") + } + if !strings.Contains(output, "actions/github-script") { + t.Fatal("expected actions/github-script usage") + } +} + +// TestGenerateGradersStep_BeforeArtifactUpload verifies ordering. +func TestGenerateGradersStep_BeforeArtifactUpload(t *testing.T) { + c := &Compiler{} + initActionPinCacheForTest(c) + var yaml strings.Builder + + data := &WorkflowData{ + Graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "retries": {ID: "retries"}, + }, + }, + } + + // Simulate the ordering: graders step then artifact upload + c.generateGradersStep(&yaml, data) + yaml.WriteString(" - name: Upload agent artifacts\n") + + output := yaml.String() + graderIdx := strings.Index(output, "Run trace graders") + uploadIdx := strings.Index(output, "Upload agent artifacts") + if graderIdx < 0 || uploadIdx < 0 { + t.Fatal("expected both steps to be present") + } + if graderIdx >= uploadIdx { + t.Fatal("graders step must come before artifact upload") + } +} + +// TestCollectGraderArtifactPaths verifies paths include manifest and results. +func TestCollectGraderArtifactPaths(t *testing.T) { + paths := collectGraderArtifactPaths() + if len(paths) != 2 { + t.Fatalf("expected 2 paths, got %d", len(paths)) + } + if !strings.Contains(paths[0], "grader_manifest.json") { + t.Fatal("expected grader_manifest.json in paths") + } + if !strings.Contains(paths[1], "grader_results.json") { + t.Fatal("expected grader_results.json in paths") + } +} + +// initActionPinCacheForTest sets up minimal action pin resolution for tests. +func initActionPinCacheForTest(c *Compiler) { + // The Compiler uses getActionPin/getCachedActionPin which resolves from a global + // cache. In tests, we just verify the step generation logic, the pin is tested separately. +} + +// TestCollectGraderArtifactPaths_AgentGradersDir verifies paths use the agent/graders subdirectory. +func TestCollectGraderArtifactPaths_AgentGradersDir(t *testing.T) { + paths := collectGraderArtifactPaths() + for _, p := range paths { + if !strings.Contains(p, "agent/graders/") { + t.Errorf("expected path to contain agent/graders/, got %q", p) + } + } +} + +// TestGenerateGraderRedactionStep_CustomOnly verifies that the redaction step is only emitted +// when custom (non-builtin) grader scripts are present. +func TestGenerateGraderRedactionStep_CustomOnly(t *testing.T) { + c := &Compiler{stepOrderTracker: NewStepOrderTracker()} + var yaml strings.Builder + + // Built-in only — no redaction step + data := &WorkflowData{ + Graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "retries": {ID: "retries"}, + }, + }, + } + c.generateGraderRedactionStep(&yaml, "", data) + if yaml.Len() > 0 { + t.Error("expected no redaction step for built-in-only graders") + } + + // Custom script — should emit redaction step + data.Graders.Graders["my-custom"] = &GraderDefinition{ + ID: "my-custom", + Script: "return {value: 1}", + } + c.generateGraderRedactionStep(&yaml, "", data) + if !strings.Contains(yaml.String(), "Redact grader outputs") { + t.Error("expected redaction step for custom grader script") + } +} + +// TestGradersConfig_FrontmatterConfigField verifies the FrontmatterConfig struct has a Graders field. +func TestGradersConfig_FrontmatterConfigField(t *testing.T) { + fc := FrontmatterConfig{ + Graders: map[string]any{}, + } + if fc.Graders == nil { + t.Error("expected Graders field to be set") + } +} diff --git a/pkg/workflow/workflow_data.go b/pkg/workflow/workflow_data.go index 73a63dd8e8c..cc558845cd2 100644 --- a/pkg/workflow/workflow_data.go +++ b/pkg/workflow/workflow_data.go @@ -206,6 +206,7 @@ type WorkflowData struct { ContainerPinMappings map[string]string // container-pin redirect table from aw.json container_pins: maps source image → replacement image GHES bool // select action versions compatible with GitHub Enterprise Server Evals *EvalsConfig // BinEval evaluation configuration parsed from frontmatter evals field + Graders *GradersConfig // Deterministic trace graders configuration parsed from frontmatter graders field ExcludedEnv []string // additional env var names to exclude from agent container via AWF --exclude-env (from frontmatter excluded-env field) } From 4b4de8a7bbaa99237df13d386f8915855cc5935e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:42:23 +0000 Subject: [PATCH 03/18] docs(adr): add draft ADR-54678 for deterministic trace grading framework Co-Authored-By: Claude Sonnet 4.6 --- ...8-deterministic-trace-grading-framework.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/adr/54678-deterministic-trace-grading-framework.md diff --git a/docs/adr/54678-deterministic-trace-grading-framework.md b/docs/adr/54678-deterministic-trace-grading-framework.md new file mode 100644 index 00000000000..c421954d0bc --- /dev/null +++ b/docs/adr/54678-deterministic-trace-grading-framework.md @@ -0,0 +1,51 @@ +# ADR-54678: Deterministic Trace Grading Framework with Single Preprocessing Pass and vm Sandbox + +**Date**: 2026-08-22 +**Status**: Draft +**Deciders**: Unknown + +--- + +### Context + +The gh-aw agent job produces execution traces (token usage JSONL, MCP gateway logs, agent output JSON) that describe how an agent ran: how many LLM requests were made, how many tool calls succeeded, how many retries occurred, and how long execution took. Before this change, no systematic, deterministic mechanism existed to compute behavioral metrics from these traces. Evaluation was ad-hoc and relied on manual inspection. The system requires metrics that are byte-identical for equivalent inputs so downstream detection jobs and artifact diffs can rely on them without accounting for nondeterminism (timestamps, randomness, or LLM stochasticity). + +### Decision + +We will implement a deterministic trace grading framework (`trace_graders.cjs`) that performs a single preprocessing pass over all trace files at the start of each grading run and shares the resulting in-memory `PreprocessedTrace` object with all graders. Built-in graders are pure functions of that object. Custom (user-supplied) inline graders run inside a `node:vm` sandbox with a frozen copy of the trace, with access to `Math`, `JSON`, `Array`, `Object`, and a small `helpers` API, and with `Date`, `Math.random`, `require`, `process`, and `fetch` excluded. Output is written to `grader_results.json` with no timestamp field, making results deterministically byte-equivalent for identical inputs. + +### Alternatives Considered + +#### Alternative 1: LLM-Based Evaluation + +Use a secondary LLM call after the agent run to grade behavior from logs. Considered because it could handle open-ended behavioral judgements beyond simple metrics. Rejected because LLM outputs are nondeterministic (same input rarely yields byte-identical output), add significant latency and cost per run, and are unavailable in sandboxed or offline environments. Determinism is a hard requirement for artifact diffing and detection staging. + +#### Alternative 2: Per-Grader File Reads + +Have each grader independently open and parse the trace files it needs. Considered because it simplifies the grader interface (each grader is fully self-contained). Rejected because it leads to redundant I/O proportional to grader count, makes it harder to enforce consistent parsing (e.g., JSONL size limits, malformed-line handling), and complicates sandboxing of custom graders (each would need its own file-access surface). The single-pass architecture keeps parsing logic in one place and is more efficient at runtime. + +#### Alternative 3: Child-Process Isolation for Custom Scripts + +Run each custom grader script in a separate child process with a restricted environment. Considered because it provides stronger OS-level isolation than `node:vm`. Rejected because `node:vm` with `codeGeneration: {strings: false, wasm: false}` and a frozen sandbox context is sufficient for the threat model (trusted repository authors running in an already-sandboxed CI environment), and avoids the latency, IPC overhead, and process-management complexity of spawning child processes per grader. The 5-second timeout and frozen trace provide adequate guardrails. + +### Consequences + +#### Positive +- Grader output (`grader_results.json`) is byte-deterministic for identical trace inputs, enabling reliable artifact diffs and detection-pipeline comparisons. +- The single preprocessing pass is O(1) in file I/O regardless of grader count — adding more graders does not add more disk reads. +- The `node:vm` sandbox prevents custom scripts from accessing the filesystem, network, or Node.js globals (`require`, `process`, `fetch`, `Date`), containing the blast radius of malicious or buggy user-supplied graders. +- Nine built-in graders (tool success rate, failure count, retries, loops, trajectory efficiency, step count, duration, context growth, artifact production) are available out of the box with no configuration. + +#### Negative +- `node:vm` is not equivalent to OS-level process isolation; V8 sandbox escapes (though rare and typically patched quickly) could theoretically allow a custom script to access the host process. Operators with stricter security requirements should audit custom grader scripts before enabling them. +- The frozen trace is deep-cloned via `JSON.parse(JSON.stringify(...))`, so non-JSON-serializable trace fields (Dates, Buffers, circular references) are silently dropped. Graders cannot receive richer data types without extending the preprocessing layer. +- Custom scripts are synchronous and bounded by a 5-second VM timeout; long-running or async computations are not supported. + +#### Neutral +- Grader output files (`grader_manifest.json`, `grader_results.json`) are written to `/tmp/gh-aw/agent/graders/` and copied into the detection staging directory, making them available to downstream jobs without changes to the artifact upload structure. +- Schema registration, canonical manifest/result metadata, and threshold configuration are deferred to follow-up work; this PR establishes the runtime and built-in grader set only. +- The grader step runs as `if: always()` after the agent, integrated via `parseGradersFromFrontmatter` in the workflow compiler — enabling opt-in via YAML frontmatter without modifying the core agent job. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* From 0d6a86e8755aae465938ae0962cb8b261e3ad4ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:43:07 +0000 Subject: [PATCH 04/18] Harden trace grader runtime isolation and manifest transport Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 133 ++++++++++-------- actions/setup/js/trace_graders_worker.cjs | 123 ++++++++++++++++ ...8-deterministic-trace-grading-framework.md | 20 +-- pkg/workflow/compiler_yaml_graders.go | 56 +++----- pkg/workflow/graders_config.go | 57 ++++---- pkg/workflow/graders_config_test.go | 25 +++- 6 files changed, 278 insertions(+), 136 deletions(-) create mode 100644 actions/setup/js/trace_graders_worker.cjs diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 6fca84159b8..7df98f8b758 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -3,7 +3,7 @@ const fs = require("fs"); const path = require("path"); -const vm = require("vm"); +const cp = require("child_process"); const crypto = require("crypto"); const { getErrorMessage } = require("./error_helpers.cjs"); @@ -29,6 +29,8 @@ const AGENT_LOG_JSONL_PATH = path.join(TMP_GH_AW, "agent_log.jsonl"); const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB const MAX_LINE_LENGTH = 1024 * 1024; // 1 MB per line const SCRIPT_TIMEOUT_MS = 5000; // 5 seconds per custom grader +const SCRIPT_WORKER_OVERHEAD_MS = 1000; // Allow worker startup/serialization overhead. +const SCRIPT_WORKER_PATH = path.join(__dirname, "trace_graders_worker.cjs"); const GRADER_VERSION = 1; const IMPLEMENTATION_ID = "gh-aw/trace-graders"; @@ -57,7 +59,7 @@ function safeReadFile(filePath) { /** * Safely parse JSONL, skipping malformed or oversized lines. * @param {string} content - * @returns {object[]} + * @returns {any[]} */ function safeParseJsonl(content) { const results = []; @@ -111,7 +113,7 @@ function deepFreeze(obj) { if (obj === null || typeof obj !== "object") return obj; Object.freeze(obj); for (const key of Object.getOwnPropertyNames(obj)) { - const v = /** @type {any} */ (obj)[key]; + const v = /** @type {any} */ obj[key]; if (v !== null && typeof v === "object" && !Object.isFrozen(v)) { deepFreeze(v); } @@ -131,21 +133,21 @@ function deepClone(obj) { /** * @typedef {object} PreprocessedTrace - * @property {object[]} tokenUsageEntries - Parsed token-usage JSONL records + * @property {any[]} tokenUsageEntries - Parsed token-usage JSONL records * @property {object|null} agentUsage - Parsed agent_usage.json - * @property {object[]} mcpGatewayEntries - Parsed MCP gateway log records + * @property {any[]} mcpGatewayEntries - Parsed MCP gateway log records * @property {object|null} agentOutput - Parsed agent_output.json - * @property {object[]} toolCalls - Extracted tool call records from MCP gateway - * @property {object[]} gatewayRequests - Request/response pairs from gateway - * @property {object[]} retryEvents - Detected retry events - * @property {object[]} errorEvents - Detected error events - * @property {object[]} steps - Extracted execution steps (LLM requests) + * @property {any[]} toolCalls - Extracted tool call records from MCP gateway + * @property {any[]} gatewayRequests - Request/response pairs from gateway + * @property {any[]} retryEvents - Detected retry events + * @property {any[]} errorEvents - Detected error events + * @property {any[]} steps - Extracted execution steps (LLM requests) * @property {number} totalInputTokens - Sum of input tokens * @property {number} totalOutputTokens - Sum of output tokens * @property {number} totalDurationMs - Sum of duration_ms from token usage * @property {number} totalRequests - Count of token usage entries (LLM requests) - * @property {object[]} files - Files mentioned in agent output - * @property {object[]} artifacts - Artifacts/outputs from agent + * @property {any[]} files - Files mentioned in agent output + * @property {any[]} artifacts - Artifacts/outputs from agent */ /** @@ -201,7 +203,7 @@ function preprocessTrace() { })); // Extract files and artifacts from agent output - const ao = /** @type {any} */ (agentOutput); + const ao = /** @type {any} */ agentOutput; const files = ao && Array.isArray(ao.files) ? ao.files : []; const artifacts = ao && Array.isArray(ao.outputs) ? ao.outputs : ao && Array.isArray(ao.items) ? ao.items : []; @@ -279,10 +281,14 @@ function gradeTrajectoryEfficiency(trace) { } /** @param {PreprocessedTrace} trace @returns {number} */ -function gradeExecutionStepCount(trace) { return trace.totalRequests; } +function gradeExecutionStepCount(trace) { + return trace.totalRequests; +} /** @param {PreprocessedTrace} trace @returns {number} */ -function gradeExecutionDuration(trace) { return trace.totalDurationMs; } +function gradeExecutionDuration(trace) { + return trace.totalDurationMs; +} /** @param {PreprocessedTrace} trace @returns {number} */ function gradeContextGrowth(trace) { @@ -428,7 +434,7 @@ function runBuiltinGrader(id, trace, meta) { } /** - * Run a custom inline script in a node:vm sandbox. + * Run a custom inline script in an isolated worker subprocess. * Script receives {trace, run, workflow, config, helpers} and should return {value, ...} or a number. * @param {string} id * @param {string} script @@ -436,47 +442,57 @@ function runBuiltinGrader(id, trace, meta) { * @param {{name: string, unit: string, direction: string, threshold?: number, source: string, digest?: string, config?: object}} meta * @returns {GraderResult} */ -function runCustomGrader(id, script, trace, meta) { +function executeCustomGraderInSubprocess(id, script, trace, meta) { + const payload = { + id, + script, + trace, + config: meta.config || {}, + timeoutMs: SCRIPT_TIMEOUT_MS, + }; + const safeEnv = {}; + for (const key of ["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "SystemRoot", "ComSpec"]) { + if (process.env[key]) { + safeEnv[key] = process.env[key]; + } + } + const timeoutMs = SCRIPT_TIMEOUT_MS + SCRIPT_WORKER_OVERHEAD_MS; + const proc = cp.spawnSync(process.execPath, [SCRIPT_WORKER_PATH], { + input: JSON.stringify(payload), + encoding: "utf-8", + timeout: timeoutMs, + maxBuffer: 1024 * 1024, + env: safeEnv, + }); + + if (proc.error) { + if (/** @type {any} */ proc.error.code === "ETIMEDOUT") { + throw new Error(`script worker timed out after ${timeoutMs}ms`); + } + throw proc.error; + } + + if (proc.status !== 0) { + const stderr = (proc.stderr || "").trim(); + throw new Error(stderr || `script worker exited with status ${String(proc.status)}`); + } + + let parsed; try { - // Build the frozen sandbox context — no require, process, fetch, Date, Math.random - const frozenTrace = deepFreeze(deepClone(trace)); - const runCtx = deepFreeze({ - graderCount: 0, // filled by caller - }); - const workflowCtx = deepFreeze({}); - const config = deepFreeze(deepClone(meta.config || {})); - const helpers = deepFreeze({ - clamp: (/** @type {number} */ v, /** @type {number} */ lo, /** @type {number} */ hi) => Math.max(lo, Math.min(hi, v)), - ratio: (/** @type {number} */ num, /** @type {number} */ den) => (den === 0 ? 0 : num / den), - sum: (/** @type {number[]} */ arr) => arr.reduce((a, b) => a + b, 0), - }); + parsed = JSON.parse(proc.stdout || "{}"); + } catch (err) { + throw new Error(`invalid script worker output: ${getErrorMessage(err)}`); + } - // Wrap script as function body - const wrappedScript = `(function(trace, run, workflow, config, helpers) { "use strict"; ${script} })`; - - const sandbox = { - Math: Object.freeze({ ...Math, random: undefined }), - JSON: Object.freeze({ parse: JSON.parse, stringify: JSON.stringify }), - Array, - Object, - String, - Number, - Boolean, - RegExp, - Map, - Set, - isFinite, - isNaN, - parseInt, - parseFloat, - undefined, - NaN, - Infinity, - }; - const ctx = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); + if (!parsed || parsed.ok !== true) { + throw new Error(parsed && typeof parsed.error === "string" ? parsed.error : "script worker returned an error"); + } + return parsed.value; +} - const fn = vm.runInContext(wrappedScript, ctx, { timeout: SCRIPT_TIMEOUT_MS, filename: `grader:${id}` }); - const rawResult = fn(frozenTrace, runCtx, workflowCtx, config, helpers); +function runCustomGrader(id, script, trace, meta) { + try { + const rawResult = executeCustomGraderInSubprocess(id, script, trace, meta); return normalizeResult(id, rawResult, meta); } catch (err) { const result = normalizeResult(id, null, meta); @@ -490,7 +506,7 @@ function runCustomGrader(id, script, trace, meta) { * Legacy adapter for existing tests. Runs a grader by id. * @param {string} id * @param {boolean} builtin - * @param {string} [script] + * @param {string|undefined} script * @param {PreprocessedTrace} trace * @param {object} [config] * @returns {{ value: number|null, error: string|null }} @@ -510,14 +526,15 @@ function runGrader(id, builtin, script, trace, config) { } /** - * Main entry point. Called from the github-script step with manifest JSON and base64 exec spec. - * @param {string} manifestJson - JSON string of grader manifest + * Main entry point. Called from the github-script step with base64 manifest and exec spec. + * @param {string} manifestB64 - Base64-encoded JSON manifest * @param {string} [execSpecB64] - Base64-encoded JSON array of {id, script} */ -async function main(manifestJson, execSpecB64) { +async function main(manifestB64, execSpecB64) { /** @type {{version: number, graders: any[]}} */ let manifest; try { + const manifestJson = Buffer.from(manifestB64, "base64").toString("utf-8"); manifest = JSON.parse(manifestJson); } catch (err) { core.setFailed(`Graders: failed to parse manifest: ${getErrorMessage(err)}`); diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs new file mode 100644 index 00000000000..6112539e230 --- /dev/null +++ b/actions/setup/js/trace_graders_worker.cjs @@ -0,0 +1,123 @@ +// @ts-check + +const vm = require("vm"); + +/** + * @param {any} obj + * @returns {any} + */ +function deepClone(obj) { + if (obj === null || obj === undefined) return obj; + return JSON.parse(JSON.stringify(obj)); +} + +/** + * @param {any} obj + * @returns {any} + */ +function deepFreeze(obj) { + if (obj === null || typeof obj !== "object") return obj; + Object.freeze(obj); + for (const key of Object.getOwnPropertyNames(obj)) { + const value = obj[key]; + if (value !== null && typeof value === "object" && !Object.isFrozen(value)) { + deepFreeze(value); + } + } + return obj; +} + +function readStdin() { + return new Promise(resolve => { + let data = ""; + process.stdin.setEncoding("utf-8"); + process.stdin.on("data", chunk => { + data += chunk; + }); + process.stdin.on("end", () => resolve(data)); + }); +} + +function makeSandboxMath() { + const math = {}; + Object.defineProperties(math, Object.getOwnPropertyDescriptors(Math)); + Object.defineProperty(math, "random", { + value: undefined, + writable: false, + enumerable: true, + configurable: false, + }); + return Object.freeze(math); +} + +async function main() { + const raw = await readStdin(); + let payload; + try { + payload = JSON.parse(raw || "{}"); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + process.stderr.write(`invalid worker payload: ${message}\n`); + process.exit(1); + } + + try { + const trace = deepFreeze(deepClone(payload.trace || {})); + const config = deepFreeze(deepClone(payload.config || {})); + const run = deepFreeze({ graderCount: 0 }); + const workflow = deepFreeze({}); + const helpers = deepFreeze({ + clamp: (v, lo, hi) => Math.max(lo, Math.min(hi, v)), + ratio: (num, den) => (den === 0 ? 0 : num / den), + sum: arr => arr.reduce((a, b) => a + b, 0), + }); + + const script = String(payload.script || ""); + const wrappedScript = ` + (() => { + const __grader = (trace, run, workflow, config, helpers) => { + "use strict"; + ${script} + }; + return __grader(trace, run, workflow, config, helpers); + })() + `; + + const sandbox = { + trace, + run, + workflow, + config, + helpers, + Math: makeSandboxMath(), + JSON: Object.freeze({ parse: JSON.parse, stringify: JSON.stringify }), + Date: undefined, + fetch: undefined, + require: undefined, + process: undefined, + global: undefined, + globalThis: undefined, + Function: undefined, + eval: undefined, + undefined, + NaN, + Infinity, + }; + const context = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); + const timeoutMs = Number(payload.timeoutMs) || 5000; + const value = vm.runInContext(wrappedScript, context, { + timeout: timeoutMs, + filename: `grader:${String(payload.id || "unknown")}`, + }); + if (typeof value === "number" && !Number.isFinite(value)) { + throw new Error("custom grader returned non-finite numeric value"); + } + + process.stdout.write(JSON.stringify({ ok: true, value })); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + process.stdout.write(JSON.stringify({ ok: false, error: message })); + } +} + +main(); diff --git a/docs/adr/54678-deterministic-trace-grading-framework.md b/docs/adr/54678-deterministic-trace-grading-framework.md index c421954d0bc..064605accca 100644 --- a/docs/adr/54678-deterministic-trace-grading-framework.md +++ b/docs/adr/54678-deterministic-trace-grading-framework.md @@ -1,8 +1,8 @@ -# ADR-54678: Deterministic Trace Grading Framework with Single Preprocessing Pass and vm Sandbox +# ADR-54678: Deterministic Trace Grading Framework with Isolated Custom Script Execution **Date**: 2026-08-22 -**Status**: Draft -**Deciders**: Unknown +**Status**: Accepted +**Deciders**: gh-aw maintainers --- @@ -12,7 +12,7 @@ The gh-aw agent job produces execution traces (token usage JSONL, MCP gateway lo ### Decision -We will implement a deterministic trace grading framework (`trace_graders.cjs`) that performs a single preprocessing pass over all trace files at the start of each grading run and shares the resulting in-memory `PreprocessedTrace` object with all graders. Built-in graders are pure functions of that object. Custom (user-supplied) inline graders run inside a `node:vm` sandbox with a frozen copy of the trace, with access to `Math`, `JSON`, `Array`, `Object`, and a small `helpers` API, and with `Date`, `Math.random`, `require`, `process`, and `fetch` excluded. Output is written to `grader_results.json` with no timestamp field, making results deterministically byte-equivalent for identical inputs. +We will implement a deterministic trace grading framework (`trace_graders.cjs`) that performs a single preprocessing pass over all trace files at the start of each grading run and shares the resulting in-memory `PreprocessedTrace` object with all graders. Built-in graders are pure functions of that object. Custom (user-supplied) inline graders execute in a dedicated worker subprocess (`trace_graders_worker.cjs`) that evaluates scripts in a restricted `node:vm` context with timeout enforcement around invocation and a serialized, frozen trace/config payload. This isolates grader execution from the main grader process and prevents direct access to host globals (`process`, `require`, `fetch`, `Date`) in the main runtime. Output is written to `grader_results.json` with no timestamp field, making results deterministically byte-equivalent for identical inputs. ### Alternatives Considered @@ -24,22 +24,22 @@ Use a secondary LLM call after the agent run to grade behavior from logs. Consid Have each grader independently open and parse the trace files it needs. Considered because it simplifies the grader interface (each grader is fully self-contained). Rejected because it leads to redundant I/O proportional to grader count, makes it harder to enforce consistent parsing (e.g., JSONL size limits, malformed-line handling), and complicates sandboxing of custom graders (each would need its own file-access surface). The single-pass architecture keeps parsing logic in one place and is more efficient at runtime. -#### Alternative 3: Child-Process Isolation for Custom Scripts +#### Alternative 3: In-Process `node:vm` Execution -Run each custom grader script in a separate child process with a restricted environment. Considered because it provides stronger OS-level isolation than `node:vm`. Rejected because `node:vm` with `codeGeneration: {strings: false, wasm: false}` and a frozen sandbox context is sufficient for the threat model (trusted repository authors running in an already-sandboxed CI environment), and avoids the latency, IPC overhead, and process-management complexity of spawning child processes per grader. The 5-second timeout and frozen trace provide adequate guardrails. +Run custom scripts directly in the main grader process with `node:vm` only. Considered because it is simpler and avoids subprocess startup overhead. Rejected because in-process execution leaves the grading controller and summary writer in the same trust boundary as untrusted script evaluation, making runtime-safety and escape-impact concerns harder to contain. We prefer an isolated subprocess boundary even with modest overhead. ### Consequences #### Positive - Grader output (`grader_results.json`) is byte-deterministic for identical trace inputs, enabling reliable artifact diffs and detection-pipeline comparisons. - The single preprocessing pass is O(1) in file I/O regardless of grader count — adding more graders does not add more disk reads. -- The `node:vm` sandbox prevents custom scripts from accessing the filesystem, network, or Node.js globals (`require`, `process`, `fetch`, `Date`), containing the blast radius of malicious or buggy user-supplied graders. +- The custom-grader subprocess boundary contains failures and sandbox escapes to a short-lived worker process, reducing impact on the main grading controller. - Nine built-in graders (tool success rate, failure count, retries, loops, trajectory efficiency, step count, duration, context growth, artifact production) are available out of the box with no configuration. #### Negative -- `node:vm` is not equivalent to OS-level process isolation; V8 sandbox escapes (though rare and typically patched quickly) could theoretically allow a custom script to access the host process. Operators with stricter security requirements should audit custom grader scripts before enabling them. +- The worker still uses `node:vm`; a sandbox escape could compromise the worker process. This is stronger than pure in-process execution but not equivalent to a hardened container or VM boundary. - The frozen trace is deep-cloned via `JSON.parse(JSON.stringify(...))`, so non-JSON-serializable trace fields (Dates, Buffers, circular references) are silently dropped. Graders cannot receive richer data types without extending the preprocessing layer. -- Custom scripts are synchronous and bounded by a 5-second VM timeout; long-running or async computations are not supported. +- Custom scripts are synchronous and bounded by timeout enforcement; long-running or async computations are not supported. #### Neutral - Grader output files (`grader_manifest.json`, `grader_results.json`) are written to `/tmp/gh-aw/agent/graders/` and copied into the detection staging directory, making them available to downstream jobs without changes to the artifact upload structure. @@ -48,4 +48,4 @@ Run each custom grader script in a separate child process with a restricted envi --- -*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* +*This ADR is accepted with follow-up work tracked in PR review threads for schema hardening and detection integration.* diff --git a/pkg/workflow/compiler_yaml_graders.go b/pkg/workflow/compiler_yaml_graders.go index aab0ee9ccea..d029dcd93aa 100644 --- a/pkg/workflow/compiler_yaml_graders.go +++ b/pkg/workflow/compiler_yaml_graders.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "sort" "strings" "github.com/github/gh-aw/pkg/constants" @@ -31,6 +32,7 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData compilerYamlGradersLog.Printf("Failed to marshal grader manifest: %v", err) return } + manifestB64 := base64.StdEncoding.EncodeToString(manifestJSON) // Build execution spec (scripts) separately, base64 encoded for safety. execSpec := buildGraderExecSpec(data.Graders) @@ -41,9 +43,6 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData } execB64 := base64.StdEncoding.EncodeToString(execJSON) - // Escape single quotes for embedding in the YAML script block - escapedManifest := strings.ReplaceAll(string(manifestJSON), "'", "\\'") - yaml.WriteString(" - name: Run trace graders\n") yaml.WriteString(" if: always()\n") yaml.WriteString(" continue-on-error: true\n") @@ -53,7 +52,7 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData yaml.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") yaml.WriteString(" const { main } = require('" + SetupActionDestination + "/trace_graders.cjs');\n") - fmt.Fprintf(yaml, " await main('%s', '%s');\n", escapedManifest, execB64) + fmt.Fprintf(yaml, " await main('%s', '%s');\n", manifestB64, execB64) compilerYamlGradersLog.Print("Generated trace graders step") } @@ -61,18 +60,18 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData // graderManifestEntry represents a single grader in the serialized manifest. // The manifest is an object {version:1, graders:[...]} for stable schema. type graderManifestEntry struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Source string `json:"source"` // "builtin" or "inline" - Enabled bool `json:"enabled"` - Unit string `json:"unit,omitempty"` - Direction string `json:"direction,omitempty"` - Threshold *float64 `json:"threshold,omitempty"` - Max *float64 `json:"max,omitempty"` - Min *float64 `json:"min,omitempty"` - Digest string `json:"digest,omitempty"` // SHA-256 of inline script - Config map[string]any `json:"config,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Source string `json:"source"` // "builtin" or "inline" + Enabled bool `json:"enabled"` + Unit string `json:"unit,omitempty"` + Direction string `json:"direction,omitempty"` + Threshold *float64 `json:"threshold,omitempty"` + Max *float64 `json:"max,omitempty"` + Min *float64 `json:"min,omitempty"` + Digest string `json:"digest,omitempty"` // SHA-256 of inline script + Config map[string]any `json:"config,omitempty"` } // graderManifest is the top-level manifest object written to disk. @@ -93,9 +92,9 @@ func buildGraderManifest(cfg *GradersConfig) *graderManifest { return &graderManifest{Version: 1} } - builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + builtinSet := make(map[string]struct{}, len(BuiltinGraderIDs)) for _, id := range BuiltinGraderIDs { - builtinSet[id] = true + builtinSet[id] = struct{}{} } ids := cfg.EnabledGraderIDs() @@ -106,14 +105,14 @@ func buildGraderManifest(cfg *GradersConfig) *graderManifest { disabledIDs = append(disabledIDs, id) } } - sortStrings(disabledIDs) + sort.Strings(disabledIDs) entries := make([]graderManifestEntry, 0, len(ids)+len(disabledIDs)) addEntry := func(id string, enabled bool) { g := cfg.Graders[id] source := "builtin" - if !builtinSet[id] { + if _, ok := builtinSet[id]; !ok { source = "inline" } name := g.Name @@ -152,32 +151,21 @@ func buildGraderExecSpec(cfg *GradersConfig) []graderExecEntry { if cfg == nil { return nil } - builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + builtinSet := make(map[string]struct{}, len(BuiltinGraderIDs)) for _, id := range BuiltinGraderIDs { - builtinSet[id] = true + builtinSet[id] = struct{}{} } var specs []graderExecEntry for _, id := range cfg.EnabledGraderIDs() { g := cfg.Graders[id] - if !builtinSet[id] && g.Script != "" { + if _, ok := builtinSet[id]; !ok && g.Script != "" { specs = append(specs, graderExecEntry{ID: id, Script: g.Script}) } } return specs } -// sortStrings sorts a string slice in place. -func sortStrings(s []string) { - for i := 0; i < len(s); i++ { - for j := i + 1; j < len(s); j++ { - if s[j] < s[i] { - s[i], s[j] = s[j], s[i] - } - } - } -} - // generateGraderRedactionStep emits a lightweight redaction pass that scans grader // output files for leaked secrets. Custom grader scripts can evaluate trace data // that may contain credential-bearing strings. This step runs after the graders step diff --git a/pkg/workflow/graders_config.go b/pkg/workflow/graders_config.go index 12924c038e6..963b656663a 100644 --- a/pkg/workflow/graders_config.go +++ b/pkg/workflow/graders_config.go @@ -27,15 +27,13 @@ type BuiltinGraderMeta struct { Min *float64 } -func ptrFloat(f float64) *float64 { return &f } - // BuiltinGraderRegistry is the ordered list of all built-in grader definitions. var BuiltinGraderRegistry = []BuiltinGraderMeta{ - {ID: "tool-success-rate", Name: "Tool Success Rate", Description: "Fraction of tool calls that succeeded", Unit: "ratio", Direction: "higher_is_better", Threshold: ptrFloat(0.8), Min: ptrFloat(0), Max: ptrFloat(1)}, - {ID: "tool-failure-count", Name: "Tool Failure Count", Description: "Number of tool calls that failed", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(5)}, - {ID: "retries", Name: "Retries", Description: "Number of retry events detected in gateway logs", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(10)}, - {ID: "loops", Name: "Loops", Description: "Consecutive identical tool calls (same name and arguments)", Unit: "count", Direction: "lower_is_better", Threshold: ptrFloat(3)}, - {ID: "trajectory-efficiency", Name: "Trajectory Efficiency", Description: "Ratio of unique tool names to total tool calls (higher = more diverse usage)", Unit: "ratio", Direction: "higher_is_better", Min: ptrFloat(0), Max: ptrFloat(1)}, + {ID: "tool-success-rate", Name: "Tool Success Rate", Description: "Fraction of tool calls that succeeded", Unit: "ratio", Direction: "higher_is_better", Threshold: new(0.8), Min: new(0.0), Max: new(1.0)}, + {ID: "tool-failure-count", Name: "Tool Failure Count", Description: "Number of tool calls that failed", Unit: "count", Direction: "lower_is_better", Threshold: new(5.0)}, + {ID: "retries", Name: "Retries", Description: "Number of retry events detected in gateway logs", Unit: "count", Direction: "lower_is_better", Threshold: new(10.0)}, + {ID: "loops", Name: "Loops", Description: "Consecutive identical tool calls (same name and arguments)", Unit: "count", Direction: "lower_is_better", Threshold: new(3.0)}, + {ID: "trajectory-efficiency", Name: "Trajectory Efficiency", Description: "Ratio of unique tool names to total tool calls (higher = more diverse usage)", Unit: "ratio", Direction: "higher_is_better", Min: new(0.0), Max: new(1.0)}, {ID: "execution-step-count", Name: "Execution Step Count", Description: "Total LLM request count", Unit: "count", Direction: "lower_is_better"}, {ID: "execution-duration", Name: "Execution Duration", Description: "Total execution duration", Unit: "ms", Direction: "lower_is_better"}, {ID: "context-growth", Name: "Context Growth", Description: "Ratio of total tokens to first-request tokens", Unit: "factor", Direction: "lower_is_better"}, @@ -62,17 +60,17 @@ var builtinGraderMetaByID = func() map[string]*BuiltinGraderMeta { // GraderDefinition represents a single grader entry in the graders map. type GraderDefinition struct { - ID string // grader identifier (must be unique) - Enabled *bool // explicit enable/disable; nil means use default (true for built-ins) - Name string // human-readable name (defaults from registry for built-ins) - Description string // description of the metric - Unit string // e.g. "ratio", "count", "ms", "factor" - Direction string // "higher_is_better" or "lower_is_better" - Threshold *float64 // quality threshold (pass/fail boundary) - Max *float64 // theoretical maximum - Min *float64 // theoretical minimum - Script string // inline JS body for trusted custom graders (built-ins leave empty) - Config map[string]any // arbitrary config passed to grader at runtime + ID string // grader identifier (must be unique) + Enabled *bool // explicit enable/disable; nil means use default (true for built-ins) + Name string // human-readable name (defaults from registry for built-ins) + Description string // description of the metric + Unit string // e.g. "ratio", "count", "ms", "factor" + Direction string // "higher_is_better" or "lower_is_better" + Threshold *float64 // quality threshold (pass/fail boundary) + Max *float64 // theoretical maximum + Min *float64 // theoretical minimum + Script string // inline JS body for trusted custom graders (built-ins leave empty) + Config map[string]any // arbitrary config passed to grader at runtime } // ScriptDigest returns the SHA-256 hex digest of the script, or "" if no script. @@ -122,24 +120,24 @@ func (gc *GradersConfig) EnabledGraderIDs() []string { if gc == nil { return nil } - enabledSet := make(map[string]bool) + enabledSet := make(map[string]struct{}) for id, g := range gc.Graders { if g.Enabled == nil || *g.Enabled { - enabledSet[id] = true + enabledSet[id] = struct{}{} } } // Stable order: built-ins first in canonical order, then custom sorted var result []string - builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + builtinSet := make(map[string]struct{}, len(BuiltinGraderIDs)) for _, bid := range BuiltinGraderIDs { - builtinSet[bid] = true - if enabledSet[bid] { + builtinSet[bid] = struct{}{} + if _, ok := enabledSet[bid]; ok { result = append(result, bid) } } var custom []string for id := range enabledSet { - if !builtinSet[id] { + if _, ok := builtinSet[id]; !ok { custom = append(custom, id) } } @@ -191,9 +189,9 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra return nil, errors.New("graders must be a map of grader IDs to configuration objects (or {} for all built-in defaults). Example:\ngraders:\n tool-success-rate:\n enabled: true") } - builtinSet := make(map[string]bool, len(BuiltinGraderIDs)) + builtinSet := make(map[string]struct{}, len(BuiltinGraderIDs)) for _, id := range BuiltinGraderIDs { - builtinSet[id] = true + builtinSet[id] = struct{}{} } // If empty map {}, populate all built-ins with defaults @@ -230,12 +228,13 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra return nil, fmt.Errorf("graders.%s must be a map or null, got %T. Example:\ngraders:\n %s:\n enabled: true", id, entryRaw, id) } - if err := parseGraderEntryFields(def, entry, id, builtinSet[id]); err != nil { + _, isBuiltin := builtinSet[id] + if err := parseGraderEntryFields(def, entry, id, isBuiltin); err != nil { return nil, err } // Custom graders must have a script - if !builtinSet[id] && def.Script == "" && (def.Enabled == nil || *def.Enabled) { + if !isBuiltin && def.Script == "" && (def.Enabled == nil || *def.Enabled) { return nil, fmt.Errorf("graders.%s is not a built-in grader and requires a 'script' field. Built-in graders: %s", id, strings.Join(BuiltinGraderIDs, ", ")) } @@ -245,7 +244,7 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra // Add missing built-ins as defaults when at least one built-in is explicitly listed hasAnyBuiltin := false for id := range cfg.Graders { - if builtinSet[id] { + if _, ok := builtinSet[id]; ok { hasAnyBuiltin = true break } diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go index 906589fe31b..09becc8ea64 100644 --- a/pkg/workflow/graders_config_test.go +++ b/pkg/workflow/graders_config_test.go @@ -224,8 +224,14 @@ func TestBuildGraderManifest(t *testing.T) { }, } entries := buildGraderManifest(cfg) - if len(entries) != 3 { - t.Fatalf("expected 3 entries, got %d", len(entries)) + if entries == nil { + t.Fatal("expected non-nil manifest") + } + if entries.Version != 1 { + t.Fatalf("expected version 1, got %d", entries.Version) + } + if len(entries.Graders) != 3 { + t.Fatalf("expected 3 entries, got %d", len(entries.Graders)) } // Verify JSON serialization round-trips @@ -233,12 +239,15 @@ func TestBuildGraderManifest(t *testing.T) { if err != nil { t.Fatalf("json marshal error: %v", err) } - var decoded []graderManifestEntry + var decoded graderManifest if err := json.Unmarshal(data, &decoded); err != nil { t.Fatalf("json unmarshal error: %v", err) } - if len(decoded) != 3 { - t.Fatalf("expected 3 decoded entries, got %d", len(decoded)) + if decoded.Version != 1 { + t.Fatalf("expected decoded version 1, got %d", decoded.Version) + } + if len(decoded.Graders) != 3 { + t.Fatalf("expected 3 decoded entries, got %d", len(decoded.Graders)) } } @@ -279,6 +288,12 @@ func TestGenerateGradersStep_Present(t *testing.T) { if !strings.Contains(output, "actions/github-script") { t.Fatal("expected actions/github-script usage") } + if !strings.Contains(output, "await main('") { + t.Fatal("expected main invocation with encoded payloads") + } + if strings.Contains(output, "{\"version\"") { + t.Fatal("expected manifest to be encoded, not embedded as raw JSON") + } } // TestGenerateGradersStep_BeforeArtifactUpload verifies ordering. From ed01ef4abef3cf4c6214ff42a2d85bd80bb6557f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:27:20 +0000 Subject: [PATCH 05/18] Refresh branch and fix trace graders JS type casts Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 7df98f8b758..e66e1d1ff0c 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -203,7 +203,7 @@ function preprocessTrace() { })); // Extract files and artifacts from agent output - const ao = /** @type {any} */ agentOutput; + const ao = agentOutput && typeof agentOutput === "object" ? /** @type {{files?: any[], outputs?: any[], items?: any[]}} */ (agentOutput) : null; const files = ao && Array.isArray(ao.files) ? ao.files : []; const artifacts = ao && Array.isArray(ao.outputs) ? ao.outputs : ao && Array.isArray(ao.items) ? ao.items : []; @@ -465,11 +465,12 @@ function executeCustomGraderInSubprocess(id, script, trace, meta) { env: safeEnv, }); - if (proc.error) { - if (/** @type {any} */ proc.error.code === "ETIMEDOUT") { + const procError = /** @type {NodeJS.ErrnoException | undefined} */ (proc.error); + if (procError) { + if (procError.code === "ETIMEDOUT") { throw new Error(`script worker timed out after ${timeoutMs}ms`); } - throw proc.error; + throw procError; } if (proc.status !== 0) { From dce434c52333559dd4523bbfb7ded335baa4dd67 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:50:57 +0000 Subject: [PATCH 06/18] Integrate experiment context into graders and evals Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/experiment_helpers.cjs | 19 ++++++- actions/setup/js/experiment_helpers.test.cjs | 8 +++ actions/setup/js/run_evals.cjs | 3 ++ actions/setup/js/run_evals.test.cjs | 21 ++++++++ actions/setup/js/trace_graders.cjs | 55 ++++++++++++++++++-- actions/setup/js/trace_graders.test.cjs | 46 +++++++++++++++- 6 files changed, 144 insertions(+), 8 deletions(-) diff --git a/actions/setup/js/experiment_helpers.cjs b/actions/setup/js/experiment_helpers.cjs index 541226dfcff..1a7c5cc115f 100644 --- a/actions/setup/js/experiment_helpers.cjs +++ b/actions/setup/js/experiment_helpers.cjs @@ -36,8 +36,25 @@ function readExperimentAssignments() { } return null; } catch { - return null; + // Fall through to environment-variable fallback. + } + + /** @type {Record} */ + const fromEnv = {}; + for (const [key, value] of Object.entries(process.env)) { + if (!key.startsWith("GH_AW_EXPERIMENTS_")) { + continue; + } + const name = key.substring("GH_AW_EXPERIMENTS_".length).toLowerCase(); + if (!name || typeof value !== "string" || value.length === 0) { + continue; + } + fromEnv[name] = value; + } + if (Object.keys(fromEnv).length > 0) { + return fromEnv; } + return null; } module.exports = { readExperimentAssignments, EXPERIMENT_ASSIGNMENTS_PATH }; diff --git a/actions/setup/js/experiment_helpers.test.cjs b/actions/setup/js/experiment_helpers.test.cjs index eca2c724d69..f13a0165c99 100644 --- a/actions/setup/js/experiment_helpers.test.cjs +++ b/actions/setup/js/experiment_helpers.test.cjs @@ -16,6 +16,8 @@ describe("readExperimentAssignments", () => { afterEach(() => { readFileSpy.mockRestore(); + delete process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE; + delete process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH; if (savedStateDir !== undefined) { process.env.GH_AW_EXPERIMENT_STATE_DIR = savedStateDir; } else { @@ -67,4 +69,10 @@ describe("readExperimentAssignments", () => { }); expect(readExperimentAssignments()).toEqual({ mode: "fast" }); }); + + it("falls back to GH_AW_EXPERIMENTS_* environment variables when assignments file is unavailable", () => { + process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE = "concise"; + process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH = "deep"; + expect(readExperimentAssignments()).toEqual({ prompt_style: "concise", reasoning_depth: "deep" }); + }); }); diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index 3360ebf788c..1e841f004ca 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -36,6 +36,7 @@ const path = require("path"); const { ERR_VALIDATION, ERR_SYSTEM } = require("./error_codes.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); +const { readExperimentAssignments } = require("./experiment_helpers.cjs"); const { resolveModelWithFallback } = require("./model_fallback.cjs"); const EVALS_DIR = "/tmp/gh-aw/evals"; @@ -127,6 +128,7 @@ async function parseMain() { const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; const model = resolveModelWithFallback(process.env, "GH_AW_EVALS_MODEL") || ""; const runID = process.env.GITHUB_RUN_ID || "unknown"; + const experimentAssignments = readExperimentAssignments(); /** @type {Array<{id: string, question: string}>} */ let questions = []; @@ -185,6 +187,7 @@ async function parseMain() { model, timestamp, runid: runID, + experiments: experimentAssignments || undefined, }; results.push(record); core.info(`Q[${q.id}]: ${answer}`); diff --git a/actions/setup/js/run_evals.test.cjs b/actions/setup/js/run_evals.test.cjs index b33ace0f170..56e59889fa7 100644 --- a/actions/setup/js/run_evals.test.cjs +++ b/actions/setup/js/run_evals.test.cjs @@ -5,6 +5,7 @@ import { createRequire } from "module"; const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`; const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; +const EXPERIMENT_ASSIGNMENTS_PATH = "/tmp/gh-aw/experiments/assignments.json"; const require = createRequire(import.meta.url); const { MODEL_FALLBACK_ENV_VAR } = require("./model_fallback.cjs"); const { setupMain, parseMain, extractAssistantTextFromJsonlLog } = require("./run_evals.cjs"); @@ -32,6 +33,9 @@ describe("run_evals.cjs", () => { if (fs.existsSync(EVALS_OUTPUT_PATH)) { fs.unlinkSync(EVALS_OUTPUT_PATH); } + if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) { + fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH); + } }); afterEach(() => { @@ -42,6 +46,9 @@ describe("run_evals.cjs", () => { if (fs.existsSync(EVALS_OUTPUT_PATH)) { fs.unlinkSync(EVALS_OUTPUT_PATH); } + if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) { + fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH); + } }); it("stores the workflow run id when writing eval records", async () => { @@ -64,6 +71,20 @@ describe("run_evals.cjs", () => { }); }); + it("includes experiment assignments in eval records when available", async () => { + vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); + vi.stubEnv("GH_AW_EVALS_MODEL", "small"); + vi.stubEnv("GITHUB_RUN_ID", "123456789"); + fs.mkdirSync("/tmp/gh-aw/experiments", { recursive: true }); + fs.writeFileSync("/tmp/gh-aw/experiments/assignments.json", JSON.stringify({ prompt_style: "concise" }) + "\n", "utf8"); + fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8"); + + await parseMain(); + + const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + expect(JSON.parse(line).experiments).toEqual({ prompt_style: "concise" }); + }); + it('falls back to "unknown" when the workflow run id is absent', async () => { vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); vi.stubEnv("GH_AW_EVALS_MODEL", "small"); diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index e66e1d1ff0c..3d61d92e4aa 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -6,6 +6,7 @@ const path = require("path"); const cp = require("child_process"); const crypto = require("crypto"); const { getErrorMessage } = require("./error_helpers.cjs"); +const { readExperimentAssignments } = require("./experiment_helpers.cjs"); // --- Constants --- const TMP_GH_AW = "/tmp/gh-aw"; @@ -24,6 +25,7 @@ const MCP_GATEWAY_LOG_PATHS = [path.join(TMP_GH_AW, "mcp-logs/gateway.jsonl"), p const AGENT_OUTPUT_PATH = path.join(TMP_GH_AW, "agent_output.json"); const AGENT_LOG_PATH = path.join(TMP_GH_AW, "agent.log"); const AGENT_LOG_JSONL_PATH = path.join(TMP_GH_AW, "agent_log.jsonl"); +const EVALS_RESULTS_PATH = path.join(TMP_GH_AW, "evals.jsonl"); // Safety limits const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB @@ -131,6 +133,44 @@ function deepClone(obj) { return JSON.parse(JSON.stringify(obj)); } +/** + * Build a compact summary of eval outputs for cross-linking with grader results. + * @returns {{total:number, yes:number, no:number, unknown:number, byQuestion:Record}|null} + */ +function readEvalSummary() { + const content = safeReadFile(EVALS_RESULTS_PATH); + if (!content) { + return null; + } + const records = safeParseJsonl(content); + if (records.length === 0) { + return null; + } + const summary = { + total: 0, + yes: 0, + no: 0, + unknown: 0, + byQuestion: {}, + }; + for (const record of records) { + if (!record || typeof record !== "object") { + continue; + } + const id = typeof record.id === "string" ? record.id : ""; + const answerRaw = typeof record.answer === "string" ? record.answer.toUpperCase() : "UNKNOWN"; + const answer = answerRaw === "YES" || answerRaw === "NO" ? answerRaw : "UNKNOWN"; + summary.total += 1; + if (answer === "YES") summary.yes += 1; + else if (answer === "NO") summary.no += 1; + else summary.unknown += 1; + if (id) { + summary.byQuestion[id] = answer; + } + } + return summary.total > 0 ? summary : null; +} + /** * @typedef {object} PreprocessedTrace * @property {any[]} tokenUsageEntries - Parsed token-usage JSONL records @@ -203,9 +243,9 @@ function preprocessTrace() { })); // Extract files and artifacts from agent output - const ao = agentOutput && typeof agentOutput === "object" ? /** @type {{files?: any[], outputs?: any[], items?: any[]}} */ (agentOutput) : null; - const files = ao && Array.isArray(ao.files) ? ao.files : []; - const artifacts = ao && Array.isArray(ao.outputs) ? ao.outputs : ao && Array.isArray(ao.items) ? ao.items : []; + const ao = agentOutput !== null && typeof agentOutput === "object" ? agentOutput : null; + const files = ao && "files" in ao && Array.isArray(ao.files) ? ao.files : []; + const artifacts = ao && "outputs" in ao && Array.isArray(ao.outputs) ? ao.outputs : ao && "items" in ao && Array.isArray(ao.items) ? ao.items : []; return { tokenUsageEntries, @@ -465,9 +505,9 @@ function executeCustomGraderInSubprocess(id, script, trace, meta) { env: safeEnv, }); - const procError = /** @type {NodeJS.ErrnoException | undefined} */ (proc.error); + const procError = proc.error; if (procError) { - if (procError.code === "ETIMEDOUT") { + if (typeof procError.message === "string" && /ETIMEDOUT|timed out/i.test(procError.message)) { throw new Error(`script worker timed out after ${timeoutMs}ms`); } throw procError; @@ -620,6 +660,10 @@ async function main(manifestB64, execSpecB64) { failed, errors: errorCount, }, + context: { + experiments: readExperimentAssignments() || undefined, + evals: readEvalSummary() || undefined, + }, results, }; @@ -668,6 +712,7 @@ module.exports = { safeParseJsonl, safeParseJson, readFirstAvailable, + readEvalSummary, deepFreeze, deepClone, runGrader, diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index bd710c0c8bb..857da02e9ec 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -12,6 +12,7 @@ const { safeParseJsonl, safeParseJson, readFirstAvailable, + readEvalSummary, deepFreeze, deepClone, runGrader, @@ -97,6 +98,39 @@ describe("trace_graders", () => { expect(safeParseJson('{"a":1}')).toEqual({ a: 1 }); }); + // --- readEvalSummary --- + describe("readEvalSummary", () => { + const evalsPath = "/tmp/gh-aw/evals.jsonl"; + + afterEach(() => { + if (fs.existsSync(evalsPath)) { + fs.unlinkSync(evalsPath); + } + }); + + it("returns null when evals file is absent", () => { + if (fs.existsSync(evalsPath)) { + fs.unlinkSync(evalsPath); + } + expect(readEvalSummary()).toBeNull(); + }); + + it("summarizes YES/NO/UNKNOWN answers", () => { + fs.writeFileSync(evalsPath, [JSON.stringify({ id: "q1", answer: "YES" }), JSON.stringify({ id: "q2", answer: "no" }), JSON.stringify({ id: "q3", answer: "MAYBE" })].join("\n") + "\n", "utf8"); + expect(readEvalSummary()).toEqual({ + total: 3, + yes: 1, + no: 1, + unknown: 1, + byQuestion: { + q1: "YES", + q2: "NO", + q3: "UNKNOWN", + }, + }); + }); + }); + it("returns null for invalid JSON", () => { expect(safeParseJson("not json")).toBeNull(); }); @@ -131,7 +165,11 @@ describe("trace_graders", () => { it("computes success rate", () => { const trace = makeTrace({ - toolCalls: [{ name: "a", success: true }, { name: "b", success: false }, { name: "c", success: true }], + toolCalls: [ + { name: "a", success: true }, + { name: "b", success: false }, + { name: "c", success: true }, + ], }); expect(gradeToolSuccessRate(trace)).toBeCloseTo(2 / 3); }); @@ -151,7 +189,11 @@ describe("trace_graders", () => { it("counts failures", () => { const trace = makeTrace({ - toolCalls: [{ name: "a", success: true }, { name: "b", success: false }, { name: "c", error: "err" }], + toolCalls: [ + { name: "a", success: true }, + { name: "b", success: false }, + { name: "c", error: "err" }, + ], }); expect(gradeToolFailureCount(trace)).toBe(2); }); From 7a02562df860aab94a2231e89347bd77d3077279 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:06:27 +0000 Subject: [PATCH 07/18] Allow experiments metrics to reference grader results Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../experimental/experiments-specification.md | 2 +- .../content/docs/experimental/experiments.md | 3 + pkg/workflow/compiler_experiments.go | 68 +++++++++-- pkg/workflow/compiler_experiments_test.go | 109 +++++++++++++++++- .../compiler_orchestrator_workflow.go | 6 +- 5 files changed, 173 insertions(+), 15 deletions(-) diff --git a/docs/src/content/docs/experimental/experiments-specification.md b/docs/src/content/docs/experimental/experiments-specification.md index 8958303e806..f3da210296a 100644 --- a/docs/src/content/docs/experimental/experiments-specification.md +++ b/docs/src/content/docs/experimental/experiments-specification.md @@ -227,7 +227,7 @@ The same minimum-two-variants constraint from R-SCHEMA-005 applies. |---|---|---| | `description` | string | Human-readable explanation of what the experiment tests. | | `hypothesis` | string | Null and alternative hypothesis statements. | -| `metric` | string | Primary metric name to observe (e.g., `aic`), or an eval reference (`eval:` / `evals.`) when using `evals:`. | +| `metric` | string | Primary metric name to observe (e.g., `aic`), or an eval reference (`eval:` / `evals.`) when using `evals:`, or a grader reference (`grader:` / `graders.`) when using `graders:`. | | `secondary_metrics` | string[] | Additional metrics to collect. | | `guardrail_metrics` | object[] | Thresholds that must not degrade. Each object has `name` (string), `threshold` (string or number), and optional `direction` (`"min"`\|`"max"`) (see §4.4). | | `min_samples` | integer ≥ 1 | Minimum runs per variant before analysis is reliable. Defaults to 20. | diff --git a/docs/src/content/docs/experimental/experiments.md b/docs/src/content/docs/experimental/experiments.md index d85a7284918..2e9336d1574 100644 --- a/docs/src/content/docs/experimental/experiments.md +++ b/docs/src/content/docs/experimental/experiments.md @@ -84,6 +84,9 @@ Summarize the findings in a **${{ experiments.prompt_style }}** way. When `evals` are configured, `metric` can reference an eval question ID using `eval:` (for example `eval:focused`) or `evals.`. +When `graders` are configured, `metric` can reference a grader result using +`grader:` (for example `grader:tool-success-rate`) or `graders.`. + `gh aw experiments analyze ` resolves the referenced eval question and, when eval result data is available, shows YES/NO/UNKNOWN totals for that eval-backed metric. diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go index 6766744dd11..b352ce75012 100644 --- a/pkg/workflow/compiler_experiments.go +++ b/pkg/workflow/compiler_experiments.go @@ -363,9 +363,34 @@ func ParseExperimentMetricEvalReference(metric string) (string, bool) { return "", false } +// ParseExperimentMetricGraderReference returns the referenced grader ID when metric +// declares a grader-backed metric. +// Supported forms: +// - grader: +// - graders. +// - graders.. (suffix reserved for future derived metrics) +func ParseExperimentMetricGraderReference(metric string) (string, bool) { + trimmed := strings.TrimSpace(metric) + if trimmed == "" { + return "", false + } + if rest, ok := strings.CutPrefix(trimmed, "grader:"); ok { + return strings.TrimSpace(rest), true + } + if rest, ok := strings.CutPrefix(trimmed, "graders."); ok { + rest = strings.TrimSpace(rest) + if rest == "" { + return "", true + } + parts := strings.SplitN(rest, ".", 2) + return parts[0], true + } + return "", false +} + // validateExperimentMetricReferences ensures experiment metrics that reference evals -// point to declared eval question IDs. -func validateExperimentMetricReferences(configs map[string]*ExperimentConfig, evals *EvalsConfig) error { +// or graders point to declared IDs. +func validateExperimentMetricReferences(configs map[string]*ExperimentConfig, evals *EvalsConfig, graders *GradersConfig) error { if len(configs) == 0 { return nil } @@ -378,6 +403,15 @@ func validateExperimentMetricReferences(configs map[string]*ExperimentConfig, ev } } } + graderIDs := map[string]struct{}{} + if graders != nil { + for id, def := range graders.Graders { + if id == "" || def == nil || (def.Enabled != nil && !*def.Enabled) { + continue + } + graderIDs[id] = struct{}{} + } + } for experimentName, cfg := range configs { if cfg == nil { @@ -393,17 +427,31 @@ func validateExperimentMetricReferences(configs map[string]*ExperimentConfig, ev } } referencedEvalID, referencesEval := ParseExperimentMetricEvalReference(metric) - if !referencesEval { + if referencesEval { + if referencedEvalID == "" { + return fmt.Errorf("experiments.%s.metric: expected eval reference format eval:; provide a declared eval question id", experimentName) + } + if _, ok := evalIDs[referencedEvalID]; !ok { + if len(evalIDs) == 0 { + return fmt.Errorf("experiments.%s.metric: references eval %q but no evals are declared", experimentName, referencedEvalID) + } + return fmt.Errorf("experiments.%s.metric: references unknown eval %q", experimentName, referencedEvalID) + } continue } - if referencedEvalID == "" { - return fmt.Errorf("experiments.%s.metric: expected eval reference format eval:; provide a declared eval question id", experimentName) - } - if _, ok := evalIDs[referencedEvalID]; !ok { - if len(evalIDs) == 0 { - return fmt.Errorf("experiments.%s.metric: references eval %q but no evals are declared", experimentName, referencedEvalID) + + referencedGraderID, referencesGrader := ParseExperimentMetricGraderReference(metric) + if referencesGrader { + if referencedGraderID == "" { + return fmt.Errorf("experiments.%s.metric: expected grader reference format grader:; provide a declared grader id", experimentName) } - return fmt.Errorf("experiments.%s.metric: references unknown eval %q", experimentName, referencedEvalID) + if _, ok := graderIDs[referencedGraderID]; !ok { + if len(graderIDs) == 0 { + return fmt.Errorf("experiments.%s.metric: references grader %q but no graders are declared", experimentName, referencedGraderID) + } + return fmt.Errorf("experiments.%s.metric: references unknown grader %q", experimentName, referencedGraderID) + } + continue } } diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go index 8d4b5df7c50..88aa3d9acd2 100644 --- a/pkg/workflow/compiler_experiments_test.go +++ b/pkg/workflow/compiler_experiments_test.go @@ -564,11 +564,37 @@ func TestParseExperimentMetricEvalReference(t *testing.T) { } } +func TestParseExperimentMetricGraderReference(t *testing.T) { + tests := []struct { + name string + metric string + wantID string + wantMatch bool + }{ + {name: "empty metric", metric: "", wantID: "", wantMatch: false}, + {name: "normal metric", metric: "aic", wantID: "", wantMatch: false}, + {name: "grader colon format", metric: "grader:loops", wantID: "loops", wantMatch: true}, + {name: "grader dotted format", metric: "graders.loops", wantID: "loops", wantMatch: true}, + {name: "grader dotted with suffix", metric: "graders.loops.value", wantID: "loops", wantMatch: true}, + {name: "grader empty id", metric: "grader:", wantID: "", wantMatch: true}, + {name: "graders empty id", metric: "graders.", wantID: "", wantMatch: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotID, gotMatch := ParseExperimentMetricGraderReference(tt.metric) + assert.Equal(t, tt.wantID, gotID) + assert.Equal(t, tt.wantMatch, gotMatch) + }) + } +} + func TestValidateExperimentMetricReferences(t *testing.T) { tests := []struct { name string configs map[string]*ExperimentConfig evals *EvalsConfig + graders *GradersConfig wantErr string }{ { @@ -577,6 +603,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { "prompt_style": {Metric: "aic"}, }, evals: nil, + graders: nil, wantErr: "", }, { @@ -587,6 +614,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { evals: &EvalsConfig{ Questions: []EvalDefinition{{ID: "builds", Question: "Does it build?"}}, }, + graders: nil, wantErr: "", }, { @@ -597,6 +625,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { evals: &EvalsConfig{ Questions: []EvalDefinition{{ID: "builds", Question: "Does it build?"}}, }, + graders: nil, wantErr: "", }, { @@ -607,6 +636,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { evals: &EvalsConfig{ Questions: []EvalDefinition{{ID: "tests", Question: "Do tests pass?"}}, }, + graders: nil, wantErr: `references unknown eval "builds"`, }, { @@ -615,6 +645,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { "prompt_style": {Metric: "eval:builds"}, }, evals: nil, + graders: nil, wantErr: `references eval "builds" but no evals are declared`, }, { @@ -625,6 +656,7 @@ func TestValidateExperimentMetricReferences(t *testing.T) { evals: &EvalsConfig{ Questions: []EvalDefinition{{ID: "builds", Question: "Does it build?"}}, }, + graders: nil, wantErr: "expected eval reference format eval:", }, { @@ -635,13 +667,88 @@ func TestValidateExperimentMetricReferences(t *testing.T) { evals: &EvalsConfig{ Questions: []EvalDefinition{{ID: "builds", Question: "Does it build?"}}, }, + graders: nil, + wantErr: "", + }, + { + name: "grader metric references existing grader", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "grader:loops"}, + }, + evals: nil, + graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "loops": {ID: "loops"}, + }, + }, + wantErr: "", + }, + { + name: "grader dotted metric references existing grader", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "graders.loops.value"}, + }, + evals: nil, + graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "loops": {ID: "loops"}, + }, + }, + wantErr: "", + }, + { + name: "grader reference rejected when grader id is unknown", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "grader:loops"}, + }, + evals: nil, + graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "retries": {ID: "retries"}, + }, + }, + wantErr: `references unknown grader "loops"`, + }, + { + name: "grader reference rejected when graders are not declared", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "grader:loops"}, + }, + evals: nil, + graders: nil, + wantErr: `references grader "loops" but no graders are declared`, + }, + { + name: "grader reference requires non empty id", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "grader:"}, + }, + evals: nil, + graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "loops": {ID: "loops"}, + }, + }, + wantErr: "expected grader reference format grader:", + }, + { + name: "grader colon metric trims whitespace from id", + configs: map[string]*ExperimentConfig{ + "prompt_style": {Metric: "grader: loops "}, + }, + evals: nil, + graders: &GradersConfig{ + Graders: map[string]*GraderDefinition{ + "loops": {ID: "loops"}, + }, + }, wantErr: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validateExperimentMetricReferences(tt.configs, tt.evals) + err := validateExperimentMetricReferences(tt.configs, tt.evals, tt.graders) if tt.wantErr == "" { assert.NoError(t, err) return diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 72e7bba742f..f43918cee05 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -560,9 +560,6 @@ func (c *Compiler) extractAdditionalConfigurations( return fmt.Errorf("invalid evals configuration: %w", err) } workflowData.Evals = evalsConfig - if err := validateExperimentMetricReferences(workflowData.ExperimentConfigs, workflowData.Evals); err != nil { - return fmt.Errorf("invalid experiments configuration: %w", err) - } // Extract deterministic trace graders configuration. gradersConfig, err := c.parseGradersFromFrontmatter(frontmatter) @@ -570,6 +567,9 @@ func (c *Compiler) extractAdditionalConfigurations( return fmt.Errorf("invalid graders configuration: %w", err) } workflowData.Graders = gradersConfig + if err := validateExperimentMetricReferences(workflowData.ExperimentConfigs, workflowData.Evals, workflowData.Graders); err != nil { + return fmt.Errorf("invalid experiments configuration: %w", err) + } return nil } From b0ad3c7c8f382002a327fe8495b3b5a6aa56609d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:21:40 +0000 Subject: [PATCH 08/18] Mark trace graders as experimental Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../content/docs/reference/trace-graders.md | 6 +- pkg/parser/schemas/main_workflow_schema.json | 66 +++++++++++++ pkg/workflow/compiler_validators.go | 1 + .../graders_experimental_warning_test.go | 99 +++++++++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 pkg/workflow/graders_experimental_warning_test.go diff --git a/docs/src/content/docs/reference/trace-graders.md b/docs/src/content/docs/reference/trace-graders.md index 2765f8a593e..68d60392e62 100644 --- a/docs/src/content/docs/reference/trace-graders.md +++ b/docs/src/content/docs/reference/trace-graders.md @@ -5,6 +5,10 @@ description: Deterministic metrics computed from agent execution traces Trace graders compute deterministic metrics from post-agent execution trace files (token usage, MCP gateway logs, agent output) without LLM calls or network access. Results are persisted in the agent artifact for downstream consumption by detection jobs and reporting tools. +:::caution[Experimental] +Trace graders are an experimental feature. +::: + ## Quick start ```yaml @@ -47,7 +51,7 @@ graders: script: "trace.toolCalls.filter(t => t.name === 'bash').length" ``` -Custom scripts must be pure expressions (≤2 KB, no `require`, `import`, `fetch`, `eval`, or `process.exit`). +Custom scripts must be pure expressions (≤4 KB, no `require`, `import`, `fetch`, `eval`, or `process.exit`). ## Output files diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 2eee9cd6050..e0ab819480b 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -12610,6 +12610,72 @@ "description": "Default LLM model, overridden by nested engine.model. Sets the default model used by the agentic engine for this workflow. Acts as a fallback when an engine instance does not specify its own 'model'; a nested 'engine.model' (e.g. safe-outputs.threat-detection.engine.model) takes precedence over this field for that engine instance. Supports full model IDs (e.g. 'claude-3-5-sonnet-20241022', 'gpt-5.4') and model aliases (e.g. 'small', 'large').", "examples": ["gpt-5.4", "claude-3-5-sonnet-20241022", "gpt-4"] }, + "graders": { + "description": "\u26a0\ufe0f Experimental. Deterministic trace graders to compute post-agent metrics from execution artifacts. Map keys are grader IDs. Built-in graders can be configured by ID; custom graders require a script.", + "type": "object", + "propertyNames": { + "pattern": "^[a-z][a-z0-9-]{0,63}$", + "description": "Grader IDs must start with a lowercase letter and contain only lowercase letters, digits, and hyphens (max 64 chars)." + }, + "additionalProperties": { + "oneOf": [ + { + "type": "null", + "description": "Enable the grader with defaults (built-ins only)." + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the grader is enabled." + }, + "name": { + "type": "string", + "description": "Optional display name override." + }, + "description": { + "type": "string", + "description": "Optional grader description." + }, + "unit": { + "type": "string", + "description": "Metric unit label such as ratio, count, ms, or factor." + }, + "direction": { + "type": "string", + "enum": ["higher_is_better", "lower_is_better"], + "description": "Optimization direction for threshold pass/fail." + }, + "threshold": { + "type": "number", + "description": "Optional threshold used for pass/fail." + }, + "min": { + "type": "number", + "description": "Optional metric minimum bound." + }, + "max": { + "type": "number", + "description": "Optional metric maximum bound." + }, + "config": { + "type": "object", + "description": "Optional grader-specific config object.", + "additionalProperties": true + }, + "script": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "description": "Custom grader JavaScript script body (trusted workflows only)." + } + } + } + ] + } + }, "evals": { "description": "\u26a0\ufe0f Experimental. BinEval binary evaluation questions to run after safe-outputs and before the conclusion job. Can be a plain list of questions (shorthand) or an object with questions, model, and runs-on fields.", "oneOf": [ diff --git a/pkg/workflow/compiler_validators.go b/pkg/workflow/compiler_validators.go index b5a941f5650..ded83737797 100644 --- a/pkg/workflow/compiler_validators.go +++ b/pkg/workflow/compiler_validators.go @@ -369,6 +369,7 @@ func (c *Compiler) emitExperimentalFeatureWarnings(workflowData *WorkflowData) { message string }{ {enabled: workflowData.RateLimit != nil, message: "Using experimental feature: rate limiting"}, + {enabled: workflowData.Graders != nil && workflowData.Graders.HasGraders(), message: "Using experimental feature: graders"}, {enabled: workflowData.SafeOutputs != nil && workflowData.SafeOutputs.DispatchRepository != nil, message: "Using experimental feature: dispatch-repository"}, {enabled: workflowData.SafeOutputs != nil && workflowData.SafeOutputs.MergePullRequest != nil, message: "Using experimental feature: merge-pull-request"}, {enabled: workflowData.SafeOutputs != nil && workflowData.SafeOutputs.ApproveWorkflowRun != nil, message: "Using experimental feature: approve-workflow-run"}, diff --git a/pkg/workflow/graders_experimental_warning_test.go b/pkg/workflow/graders_experimental_warning_test.go new file mode 100644 index 00000000000..540e2fcbb79 --- /dev/null +++ b/pkg/workflow/graders_experimental_warning_test.go @@ -0,0 +1,99 @@ +//go:build integration + +package workflow + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/github/gh-aw/pkg/testutil" +) + +func TestGradersExperimentalWarning(t *testing.T) { + tests := []struct { + name string + content string + expectWarning bool + }{ + { + name: "graders enabled produces experimental warning", + content: `--- +on: workflow_dispatch +engine: copilot +graders: {} +--- + +# Test Workflow +`, + expectWarning: true, + }, + { + name: "no graders does not produce experimental warning", + content: `--- +on: workflow_dispatch +engine: copilot +--- + +# Test Workflow +`, + expectWarning: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := testutil.TempDir(t, "graders-experimental-warning-test") + + testFile := filepath.Join(tmpDir, "test-workflow.md") + if err := os.WriteFile(testFile, []byte(tt.content), 0644); err != nil { + t.Fatal(err) + } + + oldStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = w + t.Cleanup(func() { + os.Stderr = oldStderr + }) + + compiler := NewCompiler() + compiler.SetStrictMode(false) + compileErr := compiler.CompileWorkflow(testFile) + + _ = w.Close() + os.Stderr = oldStderr + var buf bytes.Buffer + if _, copyErr := io.Copy(&buf, r); copyErr != nil { + t.Fatal(copyErr) + } + stderrOutput := buf.String() + + if compileErr != nil { + t.Errorf("expected compilation to succeed but it failed: %v", compileErr) + return + } + + expectedMessage := "Using experimental feature: graders" + if tt.expectWarning { + if !strings.Contains(stderrOutput, expectedMessage) { + t.Errorf("expected warning containing %q, got stderr:\n%s", expectedMessage, stderrOutput) + } + if compiler.GetWarningCount() == 0 { + t.Error("expected warning count > 0 but got 0") + } + return + } + + if strings.Contains(stderrOutput, expectedMessage) { + t.Errorf("did not expect warning %q, but got stderr:\n%s", expectedMessage, stderrOutput) + } + }) + } +} From 2906c4a97374b577cf7fabfcac7041eaf6b5e900 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:37:11 +0000 Subject: [PATCH 09/18] fix trace grader sandbox embedding and script-size contract Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 4 +- actions/setup/js/trace_graders.test.cjs | 13 +++ actions/setup/js/trace_graders_worker.cjs | 82 +++++++++++-------- .../content/docs/reference/trace-graders.md | 4 +- pkg/workflow/graders_config.go | 6 +- pkg/workflow/graders_config_test.go | 26 ++++++ 6 files changed, 97 insertions(+), 38 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 3d61d92e4aa..843e1196f7d 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -479,7 +479,7 @@ function runBuiltinGrader(id, trace, meta) { * @param {string} id * @param {string} script * @param {PreprocessedTrace} trace - * @param {{name: string, unit: string, direction: string, threshold?: number, source: string, digest?: string, config?: object}} meta + * @param {{name: string, unit: string, direction: string, threshold?: number, source: string, digest?: string, config?: object, graderCount?: number}} meta * @returns {GraderResult} */ function executeCustomGraderInSubprocess(id, script, trace, meta) { @@ -488,6 +488,7 @@ function executeCustomGraderInSubprocess(id, script, trace, meta) { script, trace, config: meta.config || {}, + graderCount: Number(meta.graderCount) || 0, timeoutMs: SCRIPT_TIMEOUT_MS, }; const safeEnv = {}; @@ -629,6 +630,7 @@ async function main(manifestB64, execSpecB64) { source: grader.source || "builtin", digest: grader.digest, config: grader.config, + graderCount: enabledGraders.length, }; /** @type {GraderResult} */ let result; diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index 857da02e9ec..f102a67affb 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -116,6 +116,7 @@ describe("trace_graders", () => { }); it("summarizes YES/NO/UNKNOWN answers", () => { + fs.mkdirSync(path.dirname(evalsPath), { recursive: true }); fs.writeFileSync(evalsPath, [JSON.stringify({ id: "q1", answer: "YES" }), JSON.stringify({ id: "q2", answer: "no" }), JSON.stringify({ id: "q3", answer: "MAYBE" })].join("\n") + "\n", "utf8"); expect(readEvalSummary()).toEqual({ total: 3, @@ -437,6 +438,18 @@ describe("trace_graders", () => { expect(result.value).toBe(10); }); + it("handles scripts containing template literals safely", () => { + const trace = makeTrace({ toolCalls: Array.from({ length: 12 }, () => ({ name: "a" })) }); + const result = runCustomGrader("test", "return `${trace.toolCalls.length}`.length", trace, meta); + expect(result.value).toBe(2); + expect(result.error).toBeUndefined(); + }); + + it("receives run metadata from caller", () => { + const result = runCustomGrader("test", "return run.graderCount", makeTrace(), { ...meta, graderCount: 7 }); + expect(result.value).toBe(7); + }); + it("can use helpers", () => { const result = runCustomGrader("test", "return helpers.clamp(10, 0, 5)", makeTrace(), meta); expect(result.value).toBe(5); diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs index 6112539e230..6235a05c63d 100644 --- a/actions/setup/js/trace_graders_worker.cjs +++ b/actions/setup/js/trace_graders_worker.cjs @@ -38,18 +38,6 @@ function readStdin() { }); } -function makeSandboxMath() { - const math = {}; - Object.defineProperties(math, Object.getOwnPropertyDescriptors(Math)); - Object.defineProperty(math, "random", { - value: undefined, - writable: false, - enumerable: true, - configurable: false, - }); - return Object.freeze(math); -} - async function main() { const raw = await readStdin(); let payload; @@ -64,33 +52,15 @@ async function main() { try { const trace = deepFreeze(deepClone(payload.trace || {})); const config = deepFreeze(deepClone(payload.config || {})); - const run = deepFreeze({ graderCount: 0 }); + const run = deepFreeze({ graderCount: Number(payload.graderCount) || 0 }); const workflow = deepFreeze({}); - const helpers = deepFreeze({ - clamp: (v, lo, hi) => Math.max(lo, Math.min(hi, v)), - ratio: (num, den) => (den === 0 ? 0 : num / den), - sum: arr => arr.reduce((a, b) => a + b, 0), - }); - const script = String(payload.script || ""); - const wrappedScript = ` - (() => { - const __grader = (trace, run, workflow, config, helpers) => { - "use strict"; - ${script} - }; - return __grader(trace, run, workflow, config, helpers); - })() - `; const sandbox = { trace, run, workflow, config, - helpers, - Math: makeSandboxMath(), - JSON: Object.freeze({ parse: JSON.parse, stringify: JSON.stringify }), Date: undefined, fetch: undefined, require: undefined, @@ -105,10 +75,56 @@ async function main() { }; const context = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } }); const timeoutMs = Number(payload.timeoutMs) || 5000; - const value = vm.runInContext(wrappedScript, context, { - timeout: timeoutMs, + + const runtimeBindings = vm.runInContext( + ` + "use strict"; + (() => { + const m = {}; + const descriptors = Object.getOwnPropertyDescriptors(Math); + for (const key of Object.keys(descriptors)) { + Object.defineProperty(m, key, descriptors[key]); + } + Object.defineProperty(m, "random", { + value: undefined, + writable: false, + enumerable: true, + configurable: false + }); + const safeMath = Object.freeze(m); + const helpers = Object.freeze({ + clamp: (v, lo, hi) => safeMath.max(lo, safeMath.min(hi, v)), + ratio: (num, den) => (den === 0 ? 0 : num / den), + sum: arr => arr.reduce((a, b) => a + b, 0) + }); + return { safeMath, helpers }; + })(); + `, + context, + { timeout: 1000, filename: "grader:bootstrap" } + ); + Object.defineProperty(context, "helpers", { + value: runtimeBindings.helpers, + writable: false, + enumerable: true, + configurable: false, + }); + Object.defineProperty(context, "__math", { + value: runtimeBindings.safeMath, + writable: false, + enumerable: false, + configurable: false, + }); + + const graderFn = vm.compileFunction(`"use strict";\n${script}`, ["trace", "run", "workflow", "config", "helpers", "Math"], { + parsingContext: context, filename: `grader:${String(payload.id || "unknown")}`, }); + context.__grader = graderFn; + const value = vm.runInContext("__grader(trace, run, workflow, config, helpers, __math)", context, { + timeout: timeoutMs, + filename: `grader:${String(payload.id || "unknown")}:invoke`, + }); if (typeof value === "number" && !Number.isFinite(value)) { throw new Error("custom grader returned non-finite numeric value"); } diff --git a/docs/src/content/docs/reference/trace-graders.md b/docs/src/content/docs/reference/trace-graders.md index 68d60392e62..11b802f42c9 100644 --- a/docs/src/content/docs/reference/trace-graders.md +++ b/docs/src/content/docs/reference/trace-graders.md @@ -48,10 +48,10 @@ Add a trusted inline JavaScript expression that receives the preprocessed `trace ```yaml graders: bash-calls: - script: "trace.toolCalls.filter(t => t.name === 'bash').length" + script: "return trace.toolCalls.filter(t => t.name === 'bash').length" ``` -Custom scripts must be pure expressions (≤4 KB, no `require`, `import`, `fetch`, `eval`, or `process.exit`). +Custom scripts must return a value and stay within 4096 characters (no `require`, `import`, `fetch`, `eval`, or `process.exit`). ## Output files diff --git a/pkg/workflow/graders_config.go b/pkg/workflow/graders_config.go index 963b656663a..f3c64797113 100644 --- a/pkg/workflow/graders_config.go +++ b/pkg/workflow/graders_config.go @@ -9,6 +9,7 @@ import ( "regexp" "sort" "strings" + "unicode/utf8" "github.com/github/gh-aw/pkg/logger" ) @@ -362,8 +363,9 @@ func parseGraderEntryFields(def *GraderDefinition, entry map[string]any, id stri if isBuiltin { return fmt.Errorf("graders.%s is a built-in grader and cannot have a custom script", id) } - if len(s) > 4096 { - return fmt.Errorf("graders.%s.script exceeds maximum length of 4096 characters (%d)", id, len(s)) + scriptCharCount := utf8.RuneCountInString(s) + if scriptCharCount > 4096 { + return fmt.Errorf("graders.%s.script exceeds maximum length of 4096 characters (%d)", id, scriptCharCount) } forbiddenPatterns := []string{"require(", "import(", "import ", "fetch(", "eval(", "process.exit", "child_process", "execSync", "spawnSync", "Function("} for _, p := range forbiddenPatterns { diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go index 09becc8ea64..23265572cc8 100644 --- a/pkg/workflow/graders_config_test.go +++ b/pkg/workflow/graders_config_test.go @@ -160,6 +160,32 @@ func TestParseGradersFromFrontmatter_CustomWithoutScript(t *testing.T) { } } +// TestParseGradersFromFrontmatter_ScriptLengthUsesCharacters verifies limits align to character count. +func TestParseGradersFromFrontmatter_ScriptLengthUsesCharacters(t *testing.T) { + var c Compiler + validMultibyteScript := "return '" + strings.Repeat("é", 1366) + "'.length" + if _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "unicode-ok": map[string]any{"script": validMultibyteScript}, + }, + }); err != nil { + t.Fatalf("expected multibyte script under 4096 characters to pass, got: %v", err) + } + + tooLongScript := "return '" + strings.Repeat("a", 4097) + "'.length" + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "too-long": map[string]any{"script": tooLongScript}, + }, + }) + if err == nil { + t.Fatal("expected error for script longer than 4096 characters") + } + if !strings.Contains(err.Error(), "maximum length of 4096 characters") { + t.Fatalf("expected script-length error, got: %v", err) + } +} + // TestParseGradersFromFrontmatter_InvalidID verifies ID validation. func TestParseGradersFromFrontmatter_InvalidID(t *testing.T) { var c Compiler From 803ce40aed149c0705a10e733bdfb9fd01f77f06 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:35:38 +0000 Subject: [PATCH 10/18] test: add graders integration workflow coverage Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../graders_workflow_integration_test.go | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 pkg/workflow/graders_workflow_integration_test.go diff --git a/pkg/workflow/graders_workflow_integration_test.go b/pkg/workflow/graders_workflow_integration_test.go new file mode 100644 index 00000000000..58dc4ba9f9c --- /dev/null +++ b/pkg/workflow/graders_workflow_integration_test.go @@ -0,0 +1,158 @@ +//go:build integration + +package workflow + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/github/gh-aw/pkg/stringutil" + "github.com/github/gh-aw/pkg/testutil" +) + +func TestGradersWorkflowIntegration_SyntaxParserSchema(t *testing.T) { + tmpDir := testutil.TempDir(t, "graders-workflow-integration") + workflowPath := filepath.Join(tmpDir, "workflow.md") + + content := `--- +on: workflow_dispatch +engine: copilot +strict: false +permissions: + contents: read +graders: + retries: null + custom-score: + name: Custom Score + description: Measures custom score + unit: ratio + direction: higher_is_better + threshold: 0.7 + min: 0.0 + max: 1.0 + config: + window: 5 + script: | + return helpers.clamp(trace.toolCalls.length / 10, 0, 1) +experiments: + prompt_style: + variants: [control, candidate] + metric: grader:retries + model_mix: + variants: [baseline, tuned] + metric: graders.custom-score.value +--- + +# Integration test workflow +` + require.NoError(t, os.WriteFile(workflowPath, []byte(content), 0o644)) + + compiler := NewCompiler(WithVersion("dev")) + require.NoError(t, compiler.CompileWorkflow(workflowPath)) + + compiledPath := stringutil.MarkdownToLockFile(workflowPath) + compiled, err := os.ReadFile(compiledPath) + require.NoError(t, err) + yaml := string(compiled) + + assert.Contains(t, yaml, "Run trace graders") + assert.Contains(t, yaml, "trace_graders.cjs") + assert.Contains(t, yaml, "grader:retries") + assert.Contains(t, yaml, "graders.custom-score.value") + + match := regexp.MustCompile(`await main\('([^']+)', '([^']+)'\);`).FindStringSubmatch(yaml) + require.Len(t, match, 3, "expected encoded manifest and exec spec in generated script") + + manifestJSON, err := base64.StdEncoding.DecodeString(match[1]) + require.NoError(t, err) + + var manifest graderManifest + require.NoError(t, json.Unmarshal(manifestJSON, &manifest)) + require.Equal(t, 1, manifest.Version) + + entriesByID := map[string]graderManifestEntry{} + for _, entry := range manifest.Graders { + entriesByID[entry.ID] = entry + } + + retries, ok := entriesByID["retries"] + require.True(t, ok, "expected retries grader in manifest") + assert.Equal(t, "builtin", retries.Source) + assert.True(t, retries.Enabled) + + custom, ok := entriesByID["custom-score"] + require.True(t, ok, "expected custom grader in manifest") + assert.Equal(t, "inline", custom.Source) + assert.Equal(t, "Custom Score", custom.Name) + assert.Equal(t, "ratio", custom.Unit) + assert.Equal(t, "higher_is_better", custom.Direction) + require.NotNil(t, custom.Threshold) + assert.InDelta(t, 0.7, *custom.Threshold, 0.000001) + require.NotNil(t, custom.Config) + assert.Equal(t, float64(5), custom.Config["window"]) + + execJSON, err := base64.StdEncoding.DecodeString(match[2]) + require.NoError(t, err) + + var execEntries []graderExecEntry + require.NoError(t, json.Unmarshal(execJSON, &execEntries)) + require.Len(t, execEntries, 1) + assert.Equal(t, "custom-score", execEntries[0].ID) + assert.Contains(t, execEntries[0].Script, "helpers.clamp") +} + +func TestGradersWorkflowIntegration_SchemaRejectsUnknownGraderField(t *testing.T) { + tmpDir := testutil.TempDir(t, "graders-schema-rejects-unknown-field") + workflowPath := filepath.Join(tmpDir, "workflow.md") + + content := `--- +on: workflow_dispatch +engine: copilot +graders: + retries: + unsupported: true +--- + +# Integration test workflow +` + require.NoError(t, os.WriteFile(workflowPath, []byte(content), 0o644)) + + compiler := NewCompiler(WithVersion("dev")) + err := compiler.CompileWorkflow(workflowPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "graders") + if !regexp.MustCompile(`additionalProperties|unknown field|Unknown property`).MatchString(err.Error()) { + t.Fatalf("expected schema/parser unknown-field error, got: %v", err) + } +} + +func TestGradersWorkflowIntegration_ExperimentsMetricRequiresDeclaredGrader(t *testing.T) { + tmpDir := testutil.TempDir(t, "experiments-grader-metric-reference") + workflowPath := filepath.Join(tmpDir, "workflow.md") + + content := `--- +on: workflow_dispatch +engine: copilot +strict: false +experiments: + prompt_style: + variants: [control, candidate] + metric: grader:loops +--- + +# Integration test workflow +` + require.NoError(t, os.WriteFile(workflowPath, []byte(content), 0o644)) + + compiler := NewCompiler(WithVersion("dev")) + err := compiler.CompileWorkflow(workflowPath) + require.Error(t, err) + assert.Contains(t, err.Error(), `references grader "loops" but no graders are declared`) +} From 00ca65fd075d141bae26f2bdfb16eb88ec47c04b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:16:22 +0000 Subject: [PATCH 11/18] Harden trace grader log parsing and summary output Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 57 +++++++++++++++++++---- actions/setup/js/trace_graders.test.cjs | 21 +++++++++ actions/setup/js/trace_graders_worker.cjs | 12 ++++- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 843e1196f7d..3545bc49351 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -21,7 +21,7 @@ const TOKEN_USAGE_PATHS = [ path.join(TMP_GH_AW, "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl"), ]; const AGENT_USAGE_PATH = path.join(TMP_GH_AW, "agent_usage.json"); -const MCP_GATEWAY_LOG_PATHS = [path.join(TMP_GH_AW, "mcp-logs/gateway.jsonl"), path.join(TMP_GH_AW, "mcp-logs/mcp-gateway.jsonl")]; +const MCP_GATEWAY_LOG_PATHS = [path.join(TMP_GH_AW, "mcp-logs/gateway.jsonl"), path.join(TMP_GH_AW, "mcp-logs/mcp-gateway.jsonl"), path.join(TMP_GH_AW, "mcp-logs/rpc-messages.jsonl")]; const AGENT_OUTPUT_PATH = path.join(TMP_GH_AW, "agent_output.json"); const AGENT_LOG_PATH = path.join(TMP_GH_AW, "agent.log"); const AGENT_LOG_JSONL_PATH = path.join(TMP_GH_AW, "agent_log.jsonl"); @@ -54,6 +54,7 @@ function safeReadFile(filePath) { } return fs.readFileSync(filePath, "utf-8"); } catch { + // Intentionally ignore unreadable/missing files in fallback path probes. return null; } } @@ -72,7 +73,7 @@ function safeParseJsonl(content) { try { results.push(JSON.parse(trimmed)); } catch { - // skip malformed lines + continue; } } return results; @@ -92,6 +93,14 @@ function safeParseJson(content) { } } +/** + * @param {any} v + * @returns {v is Record} + */ +function isRecord(v) { + return v !== null && typeof v === "object"; +} + /** * Read the first available file from a list of candidate paths. * @param {string[]} paths @@ -130,7 +139,11 @@ function deepFreeze(obj) { */ function deepClone(obj) { if (obj === null || obj === undefined) return obj; - return JSON.parse(JSON.stringify(obj)); + try { + return structuredClone(obj); + } catch { + return obj; + } } /** @@ -211,8 +224,24 @@ function preprocessTrace() { const agentOutputContent = safeReadFile(AGENT_OUTPUT_PATH); const agentOutput = agentOutputContent ? safeParseJson(agentOutputContent) : null; - // Extract tool calls from MCP gateway entries - const toolCalls = mcpGatewayEntries.filter(e => e.type === "tool_call" || e.method === "tools/call" || e.event === "tool_call"); + // Extract tool calls from MCP gateway entries. + // Also support rpc fallback records that expose tool_name/payload.tool_name. + const toolCalls = mcpGatewayEntries + .filter(e => { + const payload = isRecord(e.payload) ? e.payload : null; + return e.type === "tool_call" || e.method === "tools/call" || e.event === "tool_call" || typeof e.tool_name === "string" || (payload !== null && typeof payload.tool_name === "string"); + }) + .map(e => { + const payload = isRecord(e.payload) ? e.payload : null; + const toolName = typeof e.tool_name === "string" ? e.tool_name : payload !== null && typeof payload.tool_name === "string" ? payload.tool_name : undefined; + const args = payload !== null ? (payload.arguments ?? payload.params) : undefined; + return { + ...e, + name: e.name || e.tool || toolName, + tool: e.tool || toolName, + arguments: e.arguments ?? args, + }; + }); // Gateway request/response pairs const gatewayRequests = mcpGatewayEntries.filter(e => e.type === "request" || e.type === "response" || e.method); @@ -390,6 +419,18 @@ function evaluateThreshold(value, direction, threshold) { * @property {{id: string, version: number, digest?: string}} implementation */ +/** + * Escape values for step summary rendering. + * @param {any} value + * @returns {string} + */ +function sanitizeSummaryText(value) { + return String(value ?? "") + .replace(/\r?\n/g, " ") + .replace(/[<>&`]/g, ch => (ch === "<" ? "<" : ch === ">" ? ">" : ch === "&" ? "&" : "'")) + .trim(); +} + /** * Normalize a grader result from either built-in number or custom object return. * @param {string} id @@ -523,7 +564,7 @@ function executeCustomGraderInSubprocess(id, script, trace, meta) { try { parsed = JSON.parse(proc.stdout || "{}"); } catch (err) { - throw new Error(`invalid script worker output: ${getErrorMessage(err)}`); + throw new Error(`invalid script worker output: ${getErrorMessage(err)}`, { cause: err }); } if (!parsed || parsed.ok !== true) { @@ -684,7 +725,7 @@ async function main(manifestB64, execSpecB64) { const rows = tableResults.map(r => { const statusIcon = r.status === "pass" ? "✅" : r.status === "fail" ? "❌" : "⚠️"; const val = r.value !== null ? String(Number(r.value.toFixed(4))) : "—"; - return [statusIcon, r.name, r.source, val, r.unit || "—"]; + return [statusIcon, sanitizeSummaryText(r.name), r.source, val, sanitizeSummaryText(r.unit || "—")]; }); core.summary.addTable([ [ @@ -699,7 +740,7 @@ async function main(manifestB64, execSpecB64) { } const errResults = results.filter(r => r.error); if (errResults.length > 0) { - const errLines = errResults.map(r => `- **${r.id}**: ${r.error}`).join("\n"); + const errLines = errResults.map(r => `- **${sanitizeSummaryText(r.id)}**: runtime error (see step logs)`).join("\n"); core.summary.addDetails("Grader Errors", errLines); } await core.summary.write({ overwrite: false }); diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index f102a67affb..96bca660a1c 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -560,6 +560,17 @@ describe("trace_graders", () => { // --- trace.steps and enriched fields --- describe("preprocessTrace enrichment", () => { + const mcpLogsDir = "/tmp/gh-aw/mcp-logs"; + const gatewayPath = path.join(mcpLogsDir, "gateway.jsonl"); + const altGatewayPath = path.join(mcpLogsDir, "mcp-gateway.jsonl"); + const rpcMessagesPath = path.join(mcpLogsDir, "rpc-messages.jsonl"); + + afterEach(() => { + for (const p of [gatewayPath, altGatewayPath, rpcMessagesPath]) { + if (fs.existsSync(p)) fs.unlinkSync(p); + } + }); + it("extracts steps from token usage entries", () => { // Mock safeReadFile to return test data - use the preprocessTrace's logic const trace = makeTrace({ @@ -585,6 +596,16 @@ describe("trace_graders", () => { expect(trace.retryEvents.length).toBe(1); expect(trace.errorEvents.length).toBe(1); }); + + it("reads rpc-messages fallback and normalizes tool_name records", () => { + fs.mkdirSync(mcpLogsDir, { recursive: true }); + fs.writeFileSync(rpcMessagesPath, [JSON.stringify({ event: "rpc", tool_name: "github-mcp-server-search_code", payload: { tool_name: "github-mcp-server-search_code", arguments: { query: "foo" } } })].join("\n") + "\n", "utf8"); + + const trace = preprocessTrace(); + expect(trace.toolCalls.length).toBeGreaterThan(0); + expect(trace.toolCalls[0].name).toBe("github-mcp-server-search_code"); + expect(trace.toolCalls[0].arguments).toEqual({ query: "foo" }); + }); }); // --- All built-in graders are registered --- diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs index 6235a05c63d..ce97618bf95 100644 --- a/actions/setup/js/trace_graders_worker.cjs +++ b/actions/setup/js/trace_graders_worker.cjs @@ -8,7 +8,11 @@ const vm = require("vm"); */ function deepClone(obj) { if (obj === null || obj === undefined) return obj; - return JSON.parse(JSON.stringify(obj)); + try { + return structuredClone(obj); + } catch { + return obj; + } } /** @@ -136,4 +140,8 @@ async function main() { } } -main(); +main().catch(err => { + const message = err instanceof Error ? err.message : String(err); + process.stdout.write(JSON.stringify({ ok: false, error: message })); + process.exitCode = 1; +}); From 9fff955569df4c2ed53b0c2cbdb34b92503a0378 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:23:56 +0000 Subject: [PATCH 12/18] Harden trace grader fallback parsing and summary safety Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 14 ++++++++++++-- actions/setup/js/trace_graders_worker.cjs | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 3545bc49351..6d39e174eb3 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -98,7 +98,7 @@ function safeParseJson(content) { * @returns {v is Record} */ function isRecord(v) { - return v !== null && typeof v === "object"; + return v !== null && typeof v === "object" && !Array.isArray(v); } /** @@ -142,6 +142,16 @@ function deepClone(obj) { try { return structuredClone(obj); } catch { + if (Array.isArray(obj)) { + return obj.map(item => deepClone(item)); + } + if (obj !== null && typeof obj === "object") { + const out = {}; + for (const [k, v] of Object.entries(obj)) { + out[k] = deepClone(v); + } + return out; + } return obj; } } @@ -427,7 +437,7 @@ function evaluateThreshold(value, direction, threshold) { function sanitizeSummaryText(value) { return String(value ?? "") .replace(/\r?\n/g, " ") - .replace(/[<>&`]/g, ch => (ch === "<" ? "<" : ch === ">" ? ">" : ch === "&" ? "&" : "'")) + .replace(/[<>&]/g, ch => (ch === "<" ? "<" : ch === ">" ? ">" : "&")) .trim(); } diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs index ce97618bf95..6eddcbcf577 100644 --- a/actions/setup/js/trace_graders_worker.cjs +++ b/actions/setup/js/trace_graders_worker.cjs @@ -11,6 +11,16 @@ function deepClone(obj) { try { return structuredClone(obj); } catch { + if (Array.isArray(obj)) { + return obj.map(item => deepClone(item)); + } + if (obj !== null && typeof obj === "object") { + const out = {}; + for (const [k, v] of Object.entries(obj)) { + out[k] = deepClone(v); + } + return out; + } return obj; } } @@ -141,7 +151,8 @@ async function main() { } main().catch(err => { - const message = err instanceof Error ? err.message : String(err); - process.stdout.write(JSON.stringify({ ok: false, error: message })); - process.exitCode = 1; + const message = err instanceof Error ? err.stack || err.message : String(err); + process.stdout.write(JSON.stringify({ ok: false, error: message }), () => { + process.exit(1); + }); }); From 096aa3e421aa0d6678175f31f0851b9ec7d077e9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:29:18 +0000 Subject: [PATCH 13/18] chore: start triage for unresolved review feedback Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .github/workflows/agentic_commands.yml | 7 ++++--- pkg/workflow/schemas/github-workflow.json | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/agentic_commands.yml b/.github/workflows/agentic_commands.yml index dbb43b5ca6a..e9f5699b47d 100644 --- a/.github/workflows/agentic_commands.yml +++ b/.github/workflows/agentic_commands.yml @@ -1,4 +1,4 @@ -# gh-aw-commands: {"payload_version":"v1","schema_version":"v1","compiler_version":"dev","commands":["*","ace","approach-validator","archie","cloclo","craft","dependabot-burner","grumpy","matt","mergefest","nit","plan","poem-bot","ponytail","review","ruflo","scout","security-review","smoke-agent-all-merged","smoke-agent-all-none","smoke-agent-public-approved","smoke-agent-public-none","smoke-agent-scoped-approved","smoke-aider","smoke-call-workflow","smoke-checkout-pr-dispatch","smoke-claude","smoke-claude-on-copilot","smoke-codex","smoke-copilot","smoke-copilot-aoai-apikey","smoke-copilot-aoai-entra","smoke-copilot-arm","smoke-copilot-mai","smoke-copilot-sdk","smoke-copilot-small","smoke-create-cross-repo-pr","smoke-crush","smoke-cursor","smoke-deepseek-harness","smoke-gemini","smoke-github-claude","smoke-goose","smoke-kiro","smoke-multi-pr","smoke-opencode","smoke-otel-backends","smoke-pi","smoke-project","smoke-pydantic","smoke-service-ports","smoke-temporary-id","smoke-test-tools","smoke-update-cross-repo-pr","souschef","squad-plan","summarize","tidy","unbloat"],"workflows":["ace-editor","approach-validator","archie","ci-doctor","cloclo","craft","dependabot-burner","design-decision-gate","dev","grumpy-reviewer","mattpocock-skills-reviewer","mergefest","necromancer","pdf-summary","plan","poem-bot","ponytail-reviewer","pr-code-quality-reviewer","pr-nitpick-reviewer","pr-sous-chef","ruflo-backed-task","scout","security-review","skillet","smoke-agent-all-merged","smoke-agent-all-none","smoke-agent-public-approved","smoke-agent-public-none","smoke-agent-scoped-approved","smoke-aider","smoke-call-workflow","smoke-checkout-pr-dispatch","smoke-claude","smoke-claude-on-copilot","smoke-codex","smoke-copilot","smoke-copilot-aoai-apikey","smoke-copilot-aoai-entra","smoke-copilot-arm","smoke-copilot-mai","smoke-copilot-sdk","smoke-copilot-small","smoke-create-cross-repo-pr","smoke-crush","smoke-cursor","smoke-deepseek-harness","smoke-gemini","smoke-github-claude","smoke-goose","smoke-kiro","smoke-multi-pr","smoke-opencode","smoke-otel-backends","smoke-pi","smoke-project","smoke-pydantic","smoke-service-ports","smoke-temporary-id","smoke-test-tools","smoke-update-cross-repo-pr","squad-plan","test-quality-sentinel","tidy","unbloat-docs"]} +# gh-aw-commands: {"payload_version":"v1","schema_version":"v1","compiler_version":"dev","commands":["*","ace","approach-validator","archie","cloclo","craft","dependabot-burner","grumpy","matt","mergefest","nit","plan","poem-bot","ponytail","review","ruflo","scout","security-review","smoke-agent-all-merged","smoke-agent-all-none","smoke-agent-public-approved","smoke-agent-public-none","smoke-agent-scoped-approved","smoke-aider","smoke-call-workflow","smoke-checkout-pr-dispatch","smoke-claude","smoke-claude-on-copilot","smoke-codex","smoke-copilot","smoke-copilot-aoai-apikey","smoke-copilot-aoai-entra","smoke-copilot-arm","smoke-copilot-mai","smoke-copilot-sdk","smoke-copilot-small","smoke-create-cross-repo-pr","smoke-crush","smoke-cursor","smoke-deepseek-harness","smoke-drive","smoke-gemini","smoke-github-claude","smoke-goose","smoke-kiro","smoke-multi-pr","smoke-opencode","smoke-otel-backends","smoke-pi","smoke-project","smoke-pydantic","smoke-service-ports","smoke-temporary-id","smoke-test-tools","smoke-update-cross-repo-pr","souschef","squad-plan","summarize","tidy","unbloat"],"workflows":["ace-editor","approach-validator","archie","ci-doctor","cloclo","craft","dependabot-burner","design-decision-gate","dev","grumpy-reviewer","mattpocock-skills-reviewer","mergefest","necromancer","pdf-summary","plan","poem-bot","ponytail-reviewer","pr-code-quality-reviewer","pr-nitpick-reviewer","pr-sous-chef","ruflo-backed-task","scout","security-review","skillet","smoke-agent-all-merged","smoke-agent-all-none","smoke-agent-public-approved","smoke-agent-public-none","smoke-agent-scoped-approved","smoke-aider","smoke-call-workflow","smoke-checkout-pr-dispatch","smoke-claude","smoke-claude-on-copilot","smoke-codex","smoke-copilot","smoke-copilot-aoai-apikey","smoke-copilot-aoai-entra","smoke-copilot-arm","smoke-copilot-mai","smoke-copilot-sdk","smoke-copilot-small","smoke-create-cross-repo-pr","smoke-crush","smoke-cursor","smoke-deepseek-harness","smoke-drive","smoke-gemini","smoke-github-claude","smoke-goose","smoke-kiro","smoke-multi-pr","smoke-opencode","smoke-otel-backends","smoke-pi","smoke-project","smoke-pydantic","smoke-service-ports","smoke-temporary-id","smoke-test-tools","smoke-update-cross-repo-pr","squad-plan","test-quality-sentinel","tidy","unbloat-docs"]} # Routing summary (sorted): # slash commands: # /* -> skillet [pull_request_comment,pull_request_review_comment] reaction=eyes @@ -43,6 +43,7 @@ # /smoke-crush -> smoke-crush [issue_comment,issues,pull_request,pull_request_comment] reaction=eyes # /smoke-cursor -> smoke-cursor [issue_comment,issues,pull_request,pull_request_comment] reaction=rocket # /smoke-deepseek-harness -> smoke-deepseek-harness [issue_comment,issues,pull_request,pull_request_comment] reaction=eyes +# /smoke-drive -> smoke-drive [issue_comment,issues,pull_request,pull_request_comment] reaction=rocket # /smoke-gemini -> smoke-gemini [issue_comment,issues,pull_request,pull_request_comment] reaction=rocket # /smoke-github-claude -> smoke-github-claude [pull_request,pull_request_comment] reaction=eyes # /smoke-goose -> smoke-goose [issue_comment,issues,pull_request,pull_request_comment] reaction=rocket @@ -141,9 +142,9 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 # runner-guard:ignore RGS-016 -- routing tables below contain emoji variation selectors (U+FE0F) and zero-width joiners (U+200D) used to render standard emoji sequences, not steganographic payloads. env: - GH_AW_SLASH_ROUTING: '{"*":[{"workflow":"skillet","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🍳","status_comment":true}],"ace":[{"workflow":"ace-editor","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"✏️","status_comment":true}],"approach-validator":[{"workflow":"approach-validator","events":["issue_comment","pull_request_comment"],"ai_reaction":"eyes","emoji":"✅","status_comment":true}],"archie":[{"workflow":"archie","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🏛️","status_comment":true}],"cloclo":[{"workflow":"cloclo","events":["discussion","discussion_comment","issue_comment","issues","pull_request","pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"📊","status_comment":true}],"craft":[{"workflow":"craft","events":["issues"],"ai_reaction":"eyes","emoji":"✍️","status_comment":true}],"dependabot-burner":[{"workflow":"dependabot-burner","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔥","status_comment":true}],"grumpy":[{"workflow":"grumpy-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"matt":[{"workflow":"mattpocock-skills-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"mergefest":[{"workflow":"mergefest","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"🔀","status_comment":true}],"nit":[{"workflow":"pr-nitpick-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"plan":[{"workflow":"plan","events":["discussion_comment","issue_comment"],"ai_reaction":"eyes","emoji":"📋","status_comment":true}],"poem-bot":[{"workflow":"poem-bot","events":["issues"],"ai_reaction":"eyes","emoji":"🎭","status_comment":true}],"ponytail":[{"workflow":"ponytail-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"✂️","status_comment":true}],"review":[{"workflow":"design-decision-gate","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🏗️","status_comment":true},{"workflow":"pr-code-quality-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true},{"workflow":"test-quality-sentinel","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"ruflo":[{"workflow":"ruflo-backed-task","events":["issue_comment"],"ai_reaction":"eyes","status_comment":true}],"scout":[{"workflow":"scout","events":["discussion","discussion_comment","issue_comment","issues","pull_request","pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔭","status_comment":true}],"security-review":[{"workflow":"security-review","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔒","status_comment":true}],"smoke-agent-all-merged":[{"workflow":"smoke-agent-all-merged","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-all-none":[{"workflow":"smoke-agent-all-none","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-public-approved":[{"workflow":"smoke-agent-public-approved","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-public-none":[{"workflow":"smoke-agent-public-none","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-scoped-approved":[{"workflow":"smoke-agent-scoped-approved","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-aider":[{"workflow":"smoke-aider","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧑‍✈️","status_comment":true}],"smoke-call-workflow":[{"workflow":"smoke-call-workflow","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-checkout-pr-dispatch":[{"workflow":"smoke-checkout-pr-dispatch","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-claude":[{"workflow":"smoke-claude","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"heart","emoji":"🧪","status_comment":true}],"smoke-claude-on-copilot":[{"workflow":"smoke-claude-on-copilot","events":["pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-codex":[{"workflow":"smoke-codex","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"hooray","emoji":"🧪","status_comment":true}],"smoke-copilot":[{"workflow":"smoke-copilot","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-aoai-apikey":[{"workflow":"smoke-copilot-aoai-apikey","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-aoai-entra":[{"workflow":"smoke-copilot-aoai-entra","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-arm":[{"workflow":"smoke-copilot-arm","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-mai":[{"workflow":"smoke-copilot-mai","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"⚡","status_comment":true}],"smoke-copilot-sdk":[{"workflow":"smoke-copilot-sdk","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🔬","status_comment":true}],"smoke-copilot-small":[{"workflow":"smoke-copilot-small","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🪶","status_comment":true}],"smoke-create-cross-repo-pr":[{"workflow":"smoke-create-cross-repo-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-crush":[{"workflow":"smoke-crush","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-cursor":[{"workflow":"smoke-cursor","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🖱️","status_comment":true}],"smoke-deepseek-harness":[{"workflow":"smoke-deepseek-harness","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-gemini":[{"workflow":"smoke-gemini","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-github-claude":[{"workflow":"smoke-github-claude","events":["pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-goose":[{"workflow":"smoke-goose","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🪿","status_comment":true}],"smoke-kiro":[{"workflow":"smoke-kiro","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧭","status_comment":true}],"smoke-multi-pr":[{"workflow":"smoke-multi-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-opencode":[{"workflow":"smoke-opencode","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-otel-backends":[{"workflow":"smoke-otel-backends","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-pi":[{"workflow":"smoke-pi","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-project":[{"workflow":"smoke-project","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-pydantic":[{"workflow":"smoke-pydantic","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🐍","status_comment":true}],"smoke-service-ports":[{"workflow":"smoke-service-ports","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-temporary-id":[{"workflow":"smoke-temporary-id","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-test-tools":[{"workflow":"smoke-test-tools","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-update-cross-repo-pr":[{"workflow":"smoke-update-cross-repo-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"souschef":[{"workflow":"pr-sous-chef","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"👨‍🍳","status_comment":true}],"squad-plan":[{"workflow":"squad-plan","events":["issue_comment"],"ai_reaction":"eyes","emoji":"🧑‍🤝‍🧑","status_comment":true}],"summarize":[{"workflow":"pdf-summary","events":["issue_comment","issues"],"ai_reaction":"eyes","emoji":"📄","status_comment":true}],"tidy":[{"workflow":"tidy","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"🧹","status_comment":true}],"unbloat":[{"workflow":"unbloat-docs","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"📝","status_comment":true}]}' + GH_AW_SLASH_ROUTING: '{"*":[{"workflow":"skillet","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🍳","status_comment":true}],"ace":[{"workflow":"ace-editor","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"✏️","status_comment":true}],"approach-validator":[{"workflow":"approach-validator","events":["issue_comment","pull_request_comment"],"ai_reaction":"eyes","emoji":"✅","status_comment":true}],"archie":[{"workflow":"archie","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🏛️","status_comment":true}],"cloclo":[{"workflow":"cloclo","events":["discussion","discussion_comment","issue_comment","issues","pull_request","pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"📊","status_comment":true}],"craft":[{"workflow":"craft","events":["issues"],"ai_reaction":"eyes","emoji":"✍️","status_comment":true}],"dependabot-burner":[{"workflow":"dependabot-burner","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔥","status_comment":true}],"grumpy":[{"workflow":"grumpy-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"matt":[{"workflow":"mattpocock-skills-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"mergefest":[{"workflow":"mergefest","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"🔀","status_comment":true}],"nit":[{"workflow":"pr-nitpick-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true}],"plan":[{"workflow":"plan","events":["discussion_comment","issue_comment"],"ai_reaction":"eyes","emoji":"📋","status_comment":true}],"poem-bot":[{"workflow":"poem-bot","events":["issues"],"ai_reaction":"eyes","emoji":"🎭","status_comment":true}],"ponytail":[{"workflow":"ponytail-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"✂️","status_comment":true}],"review":[{"workflow":"design-decision-gate","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🏗️","status_comment":true},{"workflow":"pr-code-quality-reviewer","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔍","status_comment":true},{"workflow":"test-quality-sentinel","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"ruflo":[{"workflow":"ruflo-backed-task","events":["issue_comment"],"ai_reaction":"eyes","status_comment":true}],"scout":[{"workflow":"scout","events":["discussion","discussion_comment","issue_comment","issues","pull_request","pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔭","status_comment":true}],"security-review":[{"workflow":"security-review","events":["pull_request_comment","pull_request_review_comment"],"ai_reaction":"eyes","emoji":"🔒","status_comment":true}],"smoke-agent-all-merged":[{"workflow":"smoke-agent-all-merged","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-all-none":[{"workflow":"smoke-agent-all-none","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-public-approved":[{"workflow":"smoke-agent-public-approved","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-public-none":[{"workflow":"smoke-agent-public-none","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-agent-scoped-approved":[{"workflow":"smoke-agent-scoped-approved","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-aider":[{"workflow":"smoke-aider","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧑‍✈️","status_comment":true}],"smoke-call-workflow":[{"workflow":"smoke-call-workflow","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-checkout-pr-dispatch":[{"workflow":"smoke-checkout-pr-dispatch","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-claude":[{"workflow":"smoke-claude","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"heart","emoji":"🧪","status_comment":true}],"smoke-claude-on-copilot":[{"workflow":"smoke-claude-on-copilot","events":["pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-codex":[{"workflow":"smoke-codex","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"hooray","emoji":"🧪","status_comment":true}],"smoke-copilot":[{"workflow":"smoke-copilot","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-aoai-apikey":[{"workflow":"smoke-copilot-aoai-apikey","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-aoai-entra":[{"workflow":"smoke-copilot-aoai-entra","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-arm":[{"workflow":"smoke-copilot-arm","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-copilot-mai":[{"workflow":"smoke-copilot-mai","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"⚡","status_comment":true}],"smoke-copilot-sdk":[{"workflow":"smoke-copilot-sdk","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🔬","status_comment":true}],"smoke-copilot-small":[{"workflow":"smoke-copilot-small","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🪶","status_comment":true}],"smoke-create-cross-repo-pr":[{"workflow":"smoke-create-cross-repo-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-crush":[{"workflow":"smoke-crush","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-cursor":[{"workflow":"smoke-cursor","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🖱️","status_comment":true}],"smoke-deepseek-harness":[{"workflow":"smoke-deepseek-harness","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-drive":[{"workflow":"smoke-drive","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"💾","status_comment":true}],"smoke-gemini":[{"workflow":"smoke-gemini","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-github-claude":[{"workflow":"smoke-github-claude","events":["pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-goose":[{"workflow":"smoke-goose","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🪿","status_comment":true}],"smoke-kiro":[{"workflow":"smoke-kiro","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧭","status_comment":true}],"smoke-multi-pr":[{"workflow":"smoke-multi-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-opencode":[{"workflow":"smoke-opencode","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-otel-backends":[{"workflow":"smoke-otel-backends","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-pi":[{"workflow":"smoke-pi","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🧪","status_comment":true}],"smoke-project":[{"workflow":"smoke-project","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-pydantic":[{"workflow":"smoke-pydantic","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"rocket","emoji":"🐍","status_comment":true}],"smoke-service-ports":[{"workflow":"smoke-service-ports","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-temporary-id":[{"workflow":"smoke-temporary-id","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-test-tools":[{"workflow":"smoke-test-tools","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-update-cross-repo-pr":[{"workflow":"smoke-update-cross-repo-pr","events":["issue_comment","issues","pull_request","pull_request_comment"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"souschef":[{"workflow":"pr-sous-chef","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"👨‍🍳","status_comment":true}],"squad-plan":[{"workflow":"squad-plan","events":["issue_comment"],"ai_reaction":"eyes","emoji":"🧑‍🤝‍🧑","status_comment":true}],"summarize":[{"workflow":"pdf-summary","events":["issue_comment","issues"],"ai_reaction":"eyes","emoji":"📄","status_comment":true}],"tidy":[{"workflow":"tidy","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"🧹","status_comment":true}],"unbloat":[{"workflow":"unbloat-docs","events":["pull_request_comment"],"ai_reaction":"eyes","emoji":"📝","status_comment":true}]}' GH_AW_LABEL_ROUTING: '{"approach-proposal":[{"workflow":"approach-validator","events":["issues","pull_request"],"ai_reaction":"eyes","emoji":"✅","status_comment":true}],"ci-doctor":[{"workflow":"ci-doctor","events":["pull_request"],"ai_reaction":"eyes","emoji":"🏥","status_comment":true}],"cloclo":[{"workflow":"cloclo","events":["discussion","issues","pull_request"],"ai_reaction":"eyes","emoji":"📊","status_comment":true}],"dev":[{"workflow":"dev","events":["discussion","issues","pull_request"],"ai_reaction":"eyes","emoji":"💻","status_comment":true}],"necromancer":[{"workflow":"necromancer","events":["pull_request"],"ai_reaction":"eyes","emoji":"💀","status_comment":true}],"needs-design":[{"workflow":"approach-validator","events":["issues","pull_request"],"ai_reaction":"eyes","emoji":"✅","status_comment":true}],"smoke":[{"workflow":"smoke-copilot","events":["pull_request"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true},{"workflow":"smoke-copilot-aoai-apikey","events":["pull_request"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true},{"workflow":"smoke-copilot-aoai-entra","events":["pull_request"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true},{"workflow":"smoke-copilot-mai","events":["pull_request"],"ai_reaction":"eyes","emoji":"⚡","status_comment":true},{"workflow":"smoke-copilot-small","events":["pull_request"],"ai_reaction":"eyes","emoji":"🪶","status_comment":true},{"workflow":"smoke-otel-backends","events":["pull_request"],"ai_reaction":"eyes","emoji":"🧪","status_comment":true}],"smoke-sdk":[{"workflow":"smoke-copilot-sdk","events":["pull_request"],"ai_reaction":"eyes","emoji":"🔬","status_comment":true}]}' - GH_AW_HELP_COMMANDS: '[{"command":"*","description":"Reviews pull requests by mapping any slash command to a matching repository skill under .github/skills","centralized":true,"decentralized":false,"source_file":"skillet"},{"command":"ace","description":"Generates an ACE editor session link when invoked with /ace command on pull request comments","centralized":true,"decentralized":false,"source_file":"ace-editor"},{"command":"approach-validator","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":true,"decentralized":false,"source_file":"approach-validator"},{"command":"archie","description":"Generates Mermaid diagrams to visualize issue and pull request relationships when invoked with the /archie command","centralized":true,"decentralized":false,"source_file":"archie"},{"command":"cloclo","centralized":true,"decentralized":false,"source_file":"cloclo"},{"command":"craft","description":"Generates new agentic workflow markdown files based on user requests when invoked with /craft command","centralized":true,"decentralized":false,"source_file":"craft"},{"command":"dependabot-burner","description":"Runs one grouped Dependabot remediation wave from schedule, manual dispatch, or /dependabot-burner on pull requests","centralized":true,"decentralized":false,"source_file":"dependabot-burner"},{"command":"grumpy","description":"⚠️ DEPRECATED: Use PR Code Quality Reviewer (pr-code-quality-reviewer) instead. Performs critical code review with a focus on edge cases, potential bugs, and code quality issues","centralized":true,"decentralized":false,"source_file":"grumpy-reviewer"},{"command":"matt","description":"Reviews pull requests using Matt Pocock''s engineering skills to provide targeted, high-quality improvement suggestions based on the type of changes","centralized":true,"decentralized":false,"source_file":"mattpocock-skills-reviewer"},{"command":"mergefest","description":"Automatically merges the main branch into pull request branches when invoked with /mergefest command","centralized":true,"decentralized":false,"source_file":"mergefest"},{"command":"nit","description":"⚠️ DEPRECATED: Use PR Code Quality Reviewer (pr-code-quality-reviewer) instead. Provides detailed nitpicky code review focusing on style, best practices, and minor improvements","centralized":true,"decentralized":false,"source_file":"pr-nitpick-reviewer"},{"command":"plan","description":"Generates project plans and task breakdowns when invoked with /plan command in issues or PRs","centralized":true,"decentralized":false,"source_file":"plan"},{"command":"poem-bot","description":"Generates creative poems on specified themes when invoked with /poem-bot command","centralized":true,"decentralized":false,"source_file":"poem-bot"},{"command":"ponytail","description":"Reviews pull requests for unnecessary complexity using Ponytail","centralized":true,"decentralized":false,"source_file":"ponytail-reviewer"},{"command":"q","description":"Intelligent assistant that answers questions, analyzes repositories, and can create PRs for workflow optimizations","centralized":false,"decentralized":true,"source_file":"q"},{"command":"review","description":"Enforces Architecture Decision Records (ADRs) before implementation work can merge, detecting missing design decisions and generating draft ADRs using AI analysis","centralized":true,"decentralized":false,"source_file":"design-decision-gate"},{"command":"ruflo","description":"Runs a repository task inside GitHub Agentic Workflows while delegating inner planning and coordination to Ruflo","centralized":true,"decentralized":false,"source_file":"ruflo-backed-task"},{"command":"scout","description":"Performs deep research investigations using web search to gather and synthesize comprehensive information on any topic","centralized":true,"decentralized":false,"source_file":"scout"},{"command":"security-review","description":"Security-focused AI agent that reviews pull requests to identify changes that could weaken security posture or extend AWF boundaries","centralized":true,"decentralized":false,"source_file":"security-review"},{"command":"smoke-agent-all-merged","description":"Guard policy smoke test: repos=all, min-integrity=merged (most restrictive)","centralized":true,"decentralized":false,"source_file":"smoke-agent-all-merged"},{"command":"smoke-agent-all-none","description":"Guard policy smoke test: repos=all, min-integrity=none (most permissive)","centralized":true,"decentralized":false,"source_file":"smoke-agent-all-none"},{"command":"smoke-agent-public-approved","description":"Smoke test that validates assign-to-agent with the agentic-workflows custom agent","centralized":true,"decentralized":false,"source_file":"smoke-agent-public-approved"},{"command":"smoke-agent-public-none","description":"Guard policy smoke test: repos=public, min-integrity=none","centralized":true,"decentralized":false,"source_file":"smoke-agent-public-none"},{"command":"smoke-agent-scoped-approved","description":"Guard policy smoke test: repos=[github/gh-aw, github/*], min-integrity=approved (scoped patterns)","centralized":true,"decentralized":false,"source_file":"smoke-agent-scoped-approved"},{"command":"smoke-aider","description":"Smoke test workflow that validates Aider engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-aider"},{"command":"smoke-call-workflow","description":"Smoke test for the call-workflow safe output - orchestrator that calls a worker via workflow_call at compile-time fan-out","centralized":true,"decentralized":false,"source_file":"smoke-call-workflow"},{"command":"smoke-checkout-pr-dispatch","description":"Integration test validating that workflow_dispatch events with aw_context.item_type == ''pull_request'' correctly check out the PR branch","centralized":true,"decentralized":false,"source_file":"smoke-checkout-pr-dispatch"},{"command":"smoke-claude","description":"Smoke test workflow that validates Claude engine functionality by reviewing recent PRs twice daily","centralized":true,"decentralized":false,"source_file":"smoke-claude"},{"command":"smoke-claude-on-copilot","description":"Smoke test for Claude engine on GitHub Inference that posts a concise PR summary comment","centralized":true,"decentralized":false,"source_file":"smoke-claude-on-copilot"},{"command":"smoke-codex","description":"Smoke test workflow that validates Codex engine functionality by reviewing recent PRs twice daily","centralized":true,"decentralized":false,"source_file":"smoke-codex"},{"command":"smoke-copilot","description":"Smoke Copilot","centralized":true,"decentralized":false,"source_file":"smoke-copilot"},{"command":"smoke-copilot-aoai-apikey","description":"Smoke Copilot - AOAI (apikey)","centralized":true,"decentralized":false,"source_file":"smoke-copilot-aoai-apikey"},{"command":"smoke-copilot-aoai-entra","description":"Smoke Copilot - AOAI (Entra)","centralized":true,"decentralized":false,"source_file":"smoke-copilot-aoai-entra"},{"command":"smoke-copilot-arm","description":"Smoke Copilot ARM64","centralized":true,"decentralized":false,"source_file":"smoke-copilot-arm"},{"command":"smoke-copilot-mai","description":"Smoke test for MAI-Code-1-Flash (mai-code-1-flash-picker) — pricing: $0.75/M input, $0.075/M cached, $4.50/M output","centralized":true,"decentralized":false,"source_file":"smoke-copilot-mai"},{"command":"smoke-copilot-sdk","description":"Smoke Copilot SDK","centralized":true,"decentralized":false,"source_file":"smoke-copilot-sdk"},{"command":"smoke-copilot-small","description":"Smoke Copilot Small","centralized":true,"decentralized":false,"source_file":"smoke-copilot-small"},{"command":"smoke-create-cross-repo-pr","description":"Smoke test validating cross-repo pull request creation in github/gh-aw-side-repo","centralized":true,"decentralized":false,"source_file":"smoke-create-cross-repo-pr"},{"command":"smoke-crush","description":"Smoke test workflow that validates Crush engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-crush"},{"command":"smoke-cursor","description":"Smoke test workflow that validates Cursor engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-cursor"},{"command":"smoke-deepseek-harness","description":"Smoke test workflow that validates DeepSeek Harness engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-deepseek-harness"},{"command":"smoke-gemini","description":"Smoke test workflow that validates Gemini engine functionality twice daily","centralized":true,"decentralized":false,"source_file":"smoke-gemini"},{"command":"smoke-github-claude","description":"Smoke test for Claude engine using GitHub provider that posts a concise PR summary comment","centralized":true,"decentralized":false,"source_file":"smoke-github-claude"},{"command":"smoke-goose","description":"Smoke test workflow that validates Goose engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-goose"},{"command":"smoke-kiro","description":"Smoke test workflow that validates Kiro engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-kiro"},{"command":"smoke-multi-pr","description":"Test creating multiple pull requests in a single workflow run","centralized":true,"decentralized":false,"source_file":"smoke-multi-pr"},{"command":"smoke-opencode","description":"Smoke test workflow that validates OpenCode engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-opencode"},{"command":"smoke-otel-backends","description":"Smoke test that validates OTEL span export and query access for Sentry, Grafana, and Datadog","centralized":true,"decentralized":false,"source_file":"smoke-otel-backends"},{"command":"smoke-pi","description":"Smoke test workflow that validates Pi engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-pi"},{"command":"smoke-project","description":"Smoke Project - Test project operations","centralized":true,"decentralized":false,"source_file":"smoke-project"},{"command":"smoke-pydantic","description":"Smoke test workflow that validates Pydantic AI engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-pydantic"},{"command":"smoke-service-ports","description":"Smoke test to validate --allow-host-service-ports with Redis service container","centralized":true,"decentralized":false,"source_file":"smoke-service-ports"},{"command":"smoke-temporary-id","description":"Test temporary ID functionality for issue chaining and cross-references","centralized":true,"decentralized":false,"source_file":"smoke-temporary-id"},{"command":"smoke-test-tools","description":"Smoke test to validate common development tools are available in the agent container","centralized":true,"decentralized":false,"source_file":"smoke-test-tools"},{"command":"smoke-update-cross-repo-pr","description":"Smoke test validating cross-repo pull request updates in github/gh-aw-side-repo by adding lines from Homer''s Odyssey to the README","centralized":true,"decentralized":false,"source_file":"smoke-update-cross-repo-pr"},{"command":"souschef","description":"Keeps open non-draft PRs moving toward maintainer investigation by posting targeted Copilot nudges","centralized":true,"decentralized":false,"source_file":"pr-sous-chef"},{"command":"squad","description":"Cast, connect, or adopt a Squad AI team for your repository","centralized":false,"decentralized":true,"source_file":"squad"},{"command":"squad-plan","description":"Uses Squad to plan an issue from the /squad-plan slash command and create Copilot-ready sub-issues","centralized":true,"decentralized":false,"source_file":"squad-plan"},{"command":"summarize","description":"pdf summarizer","centralized":true,"decentralized":false,"source_file":"pdf-summary"},{"command":"tidy","description":"Automatically formats and tidies code files (Go, JS, TypeScript) on schedule or command","centralized":true,"decentralized":false,"source_file":"tidy"},{"command":"unbloat","description":"Reviews and simplifies documentation by reducing verbosity while maintaining clarity and completeness","centralized":true,"decentralized":false,"source_file":"unbloat-docs"},{"command":"approach-proposal","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":false,"decentralized":false,"label":true,"source_file":"approach-validator"},{"command":"ci-doctor","description":"Investigates failed CI workflows to identify root causes and patterns, creating issues with diagnostic information; also reviews PR check failures when the ci-doctor label is applied","centralized":false,"decentralized":false,"label":true,"source_file":"ci-doctor"},{"command":"cloclo","centralized":false,"decentralized":false,"label":true,"source_file":"cloclo"},{"command":"dev","description":"Daily status report for gh-aw project","centralized":false,"decentralized":false,"label":true,"source_file":"dev"},{"command":"necromancer","description":"Investigates merge-ready pull requests, traces root-cause issues, and adds regression tests before merge","centralized":false,"decentralized":false,"label":true,"source_file":"necromancer"},{"command":"needs-design","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":false,"decentralized":false,"label":true,"source_file":"approach-validator"},{"command":"smoke","description":"Smoke Copilot - AOAI (apikey)","centralized":false,"decentralized":false,"label":true,"source_file":"smoke-copilot-aoai-apikey"},{"command":"smoke-sdk","description":"Smoke Copilot SDK","centralized":false,"decentralized":false,"label":true,"source_file":"smoke-copilot-sdk"}]' + GH_AW_HELP_COMMANDS: '[{"command":"*","description":"Reviews pull requests by mapping any slash command to a matching repository skill under .github/skills","centralized":true,"decentralized":false,"source_file":"skillet"},{"command":"ace","description":"Generates an ACE editor session link when invoked with /ace command on pull request comments","centralized":true,"decentralized":false,"source_file":"ace-editor"},{"command":"approach-validator","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":true,"decentralized":false,"source_file":"approach-validator"},{"command":"archie","description":"Generates Mermaid diagrams to visualize issue and pull request relationships when invoked with the /archie command","centralized":true,"decentralized":false,"source_file":"archie"},{"command":"cloclo","centralized":true,"decentralized":false,"source_file":"cloclo"},{"command":"craft","description":"Generates new agentic workflow markdown files based on user requests when invoked with /craft command","centralized":true,"decentralized":false,"source_file":"craft"},{"command":"dependabot-burner","description":"Runs one grouped Dependabot remediation wave from schedule, manual dispatch, or /dependabot-burner on pull requests","centralized":true,"decentralized":false,"source_file":"dependabot-burner"},{"command":"grumpy","description":"⚠️ DEPRECATED: Use PR Code Quality Reviewer (pr-code-quality-reviewer) instead. Performs critical code review with a focus on edge cases, potential bugs, and code quality issues","centralized":true,"decentralized":false,"source_file":"grumpy-reviewer"},{"command":"matt","description":"Reviews pull requests using Matt Pocock''s engineering skills to provide targeted, high-quality improvement suggestions based on the type of changes","centralized":true,"decentralized":false,"source_file":"mattpocock-skills-reviewer"},{"command":"mergefest","description":"Automatically merges the main branch into pull request branches when invoked with /mergefest command","centralized":true,"decentralized":false,"source_file":"mergefest"},{"command":"nit","description":"⚠️ DEPRECATED: Use PR Code Quality Reviewer (pr-code-quality-reviewer) instead. Provides detailed nitpicky code review focusing on style, best practices, and minor improvements","centralized":true,"decentralized":false,"source_file":"pr-nitpick-reviewer"},{"command":"plan","description":"Generates project plans and task breakdowns when invoked with /plan command in issues or PRs","centralized":true,"decentralized":false,"source_file":"plan"},{"command":"poem-bot","description":"Generates creative poems on specified themes when invoked with /poem-bot command","centralized":true,"decentralized":false,"source_file":"poem-bot"},{"command":"ponytail","description":"Reviews pull requests for unnecessary complexity using Ponytail","centralized":true,"decentralized":false,"source_file":"ponytail-reviewer"},{"command":"q","description":"Intelligent assistant that answers questions, analyzes repositories, and can create PRs for workflow optimizations","centralized":false,"decentralized":true,"source_file":"q"},{"command":"review","description":"Enforces Architecture Decision Records (ADRs) before implementation work can merge, detecting missing design decisions and generating draft ADRs using AI analysis","centralized":true,"decentralized":false,"source_file":"design-decision-gate"},{"command":"ruflo","description":"Runs a repository task inside GitHub Agentic Workflows while delegating inner planning and coordination to Ruflo","centralized":true,"decentralized":false,"source_file":"ruflo-backed-task"},{"command":"scout","description":"Performs deep research investigations using web search to gather and synthesize comprehensive information on any topic","centralized":true,"decentralized":false,"source_file":"scout"},{"command":"security-review","description":"Security-focused AI agent that reviews pull requests to identify changes that could weaken security posture or extend AWF boundaries","centralized":true,"decentralized":false,"source_file":"security-review"},{"command":"smoke-agent-all-merged","description":"Guard policy smoke test: repos=all, min-integrity=merged (most restrictive)","centralized":true,"decentralized":false,"source_file":"smoke-agent-all-merged"},{"command":"smoke-agent-all-none","description":"Guard policy smoke test: repos=all, min-integrity=none (most permissive)","centralized":true,"decentralized":false,"source_file":"smoke-agent-all-none"},{"command":"smoke-agent-public-approved","description":"Smoke test that validates assign-to-agent with the agentic-workflows custom agent","centralized":true,"decentralized":false,"source_file":"smoke-agent-public-approved"},{"command":"smoke-agent-public-none","description":"Guard policy smoke test: repos=public, min-integrity=none","centralized":true,"decentralized":false,"source_file":"smoke-agent-public-none"},{"command":"smoke-agent-scoped-approved","description":"Guard policy smoke test: repos=[github/gh-aw, github/*], min-integrity=approved (scoped patterns)","centralized":true,"decentralized":false,"source_file":"smoke-agent-scoped-approved"},{"command":"smoke-aider","description":"Smoke test workflow that validates Aider engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-aider"},{"command":"smoke-call-workflow","description":"Smoke test for the call-workflow safe output - orchestrator that calls a worker via workflow_call at compile-time fan-out","centralized":true,"decentralized":false,"source_file":"smoke-call-workflow"},{"command":"smoke-checkout-pr-dispatch","description":"Integration test validating that workflow_dispatch events with aw_context.item_type == ''pull_request'' correctly check out the PR branch","centralized":true,"decentralized":false,"source_file":"smoke-checkout-pr-dispatch"},{"command":"smoke-claude","description":"Smoke test workflow that validates Claude engine functionality by reviewing recent PRs twice daily","centralized":true,"decentralized":false,"source_file":"smoke-claude"},{"command":"smoke-claude-on-copilot","description":"Smoke test for Claude engine on GitHub Inference that posts a concise PR summary comment","centralized":true,"decentralized":false,"source_file":"smoke-claude-on-copilot"},{"command":"smoke-codex","description":"Smoke test workflow that validates Codex engine functionality by reviewing recent PRs twice daily","centralized":true,"decentralized":false,"source_file":"smoke-codex"},{"command":"smoke-copilot","description":"Smoke Copilot","centralized":true,"decentralized":false,"source_file":"smoke-copilot"},{"command":"smoke-copilot-aoai-apikey","description":"Smoke Copilot - AOAI (apikey)","centralized":true,"decentralized":false,"source_file":"smoke-copilot-aoai-apikey"},{"command":"smoke-copilot-aoai-entra","description":"Smoke Copilot - AOAI (Entra)","centralized":true,"decentralized":false,"source_file":"smoke-copilot-aoai-entra"},{"command":"smoke-copilot-arm","description":"Smoke Copilot ARM64","centralized":true,"decentralized":false,"source_file":"smoke-copilot-arm"},{"command":"smoke-copilot-mai","description":"Smoke test for MAI-Code-1-Flash (mai-code-1-flash-picker) — pricing: $0.75/M input, $0.075/M cached, $4.50/M output","centralized":true,"decentralized":false,"source_file":"smoke-copilot-mai"},{"command":"smoke-copilot-sdk","description":"Smoke Copilot SDK","centralized":true,"decentralized":false,"source_file":"smoke-copilot-sdk"},{"command":"smoke-copilot-small","description":"Smoke Copilot Small","centralized":true,"decentralized":false,"source_file":"smoke-copilot-small"},{"command":"smoke-create-cross-repo-pr","description":"Smoke test validating cross-repo pull request creation in github/gh-aw-side-repo","centralized":true,"decentralized":false,"source_file":"smoke-create-cross-repo-pr"},{"command":"smoke-crush","description":"Smoke test workflow that validates Crush engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-crush"},{"command":"smoke-cursor","description":"Smoke test workflow that validates Cursor engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-cursor"},{"command":"smoke-deepseek-harness","description":"Smoke test workflow that validates DeepSeek Harness engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-deepseek-harness"},{"command":"smoke-drive","description":"Smoke test workflow that validates experimental GitHub Drives memory","centralized":true,"decentralized":false,"source_file":"smoke-drive"},{"command":"smoke-gemini","description":"Smoke test workflow that validates Gemini engine functionality twice daily","centralized":true,"decentralized":false,"source_file":"smoke-gemini"},{"command":"smoke-github-claude","description":"Smoke test for Claude engine using GitHub provider that posts a concise PR summary comment","centralized":true,"decentralized":false,"source_file":"smoke-github-claude"},{"command":"smoke-goose","description":"Smoke test workflow that validates Goose engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-goose"},{"command":"smoke-kiro","description":"Smoke test workflow that validates Kiro engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-kiro"},{"command":"smoke-multi-pr","description":"Test creating multiple pull requests in a single workflow run","centralized":true,"decentralized":false,"source_file":"smoke-multi-pr"},{"command":"smoke-opencode","description":"Smoke test workflow that validates OpenCode engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-opencode"},{"command":"smoke-otel-backends","description":"Smoke test that validates OTEL span export and query access for Sentry, Grafana, and Datadog","centralized":true,"decentralized":false,"source_file":"smoke-otel-backends"},{"command":"smoke-pi","description":"Smoke test workflow that validates Pi engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-pi"},{"command":"smoke-project","description":"Smoke Project - Test project operations","centralized":true,"decentralized":false,"source_file":"smoke-project"},{"command":"smoke-pydantic","description":"Smoke test workflow that validates Pydantic AI engine functionality","centralized":true,"decentralized":false,"source_file":"smoke-pydantic"},{"command":"smoke-service-ports","description":"Smoke test to validate --allow-host-service-ports with Redis service container","centralized":true,"decentralized":false,"source_file":"smoke-service-ports"},{"command":"smoke-temporary-id","description":"Test temporary ID functionality for issue chaining and cross-references","centralized":true,"decentralized":false,"source_file":"smoke-temporary-id"},{"command":"smoke-test-tools","description":"Smoke test to validate common development tools are available in the agent container","centralized":true,"decentralized":false,"source_file":"smoke-test-tools"},{"command":"smoke-update-cross-repo-pr","description":"Smoke test validating cross-repo pull request updates in github/gh-aw-side-repo by adding lines from Homer''s Odyssey to the README","centralized":true,"decentralized":false,"source_file":"smoke-update-cross-repo-pr"},{"command":"souschef","description":"Keeps open non-draft PRs moving toward maintainer investigation by posting targeted Copilot nudges","centralized":true,"decentralized":false,"source_file":"pr-sous-chef"},{"command":"squad","description":"Cast, connect, or adopt a Squad AI team for your repository","centralized":false,"decentralized":true,"source_file":"squad"},{"command":"squad-plan","description":"Uses Squad to plan an issue from the /squad-plan slash command and create Copilot-ready sub-issues","centralized":true,"decentralized":false,"source_file":"squad-plan"},{"command":"summarize","description":"pdf summarizer","centralized":true,"decentralized":false,"source_file":"pdf-summary"},{"command":"tidy","description":"Automatically formats and tidies code files (Go, JS, TypeScript) on schedule or command","centralized":true,"decentralized":false,"source_file":"tidy"},{"command":"unbloat","description":"Reviews and simplifies documentation by reducing verbosity while maintaining clarity and completeness","centralized":true,"decentralized":false,"source_file":"unbloat-docs"},{"command":"approach-proposal","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":false,"decentralized":false,"label":true,"source_file":"approach-validator"},{"command":"ci-doctor","description":"Investigates failed CI workflows to identify root causes and patterns, creating issues with diagnostic information; also reviews PR check failures when the ci-doctor label is applied","centralized":false,"decentralized":false,"label":true,"source_file":"ci-doctor"},{"command":"cloclo","centralized":false,"decentralized":false,"label":true,"source_file":"cloclo"},{"command":"dev","description":"Daily status report for gh-aw project","centralized":false,"decentralized":false,"label":true,"source_file":"dev"},{"command":"necromancer","description":"Investigates merge-ready pull requests, traces root-cause issues, and adds regression tests before merge","centralized":false,"decentralized":false,"label":true,"source_file":"necromancer"},{"command":"needs-design","description":"Validates proposed technical approaches before implementation begins using a sequential multi-agent panel of Devil''s Advocate, Alternatives Scout, Implementation Estimator, and Dead End Detector","centralized":false,"decentralized":false,"label":true,"source_file":"approach-validator"},{"command":"smoke","description":"Smoke Copilot - AOAI (apikey)","centralized":false,"decentralized":false,"label":true,"source_file":"smoke-copilot-aoai-apikey"},{"command":"smoke-sdk","description":"Smoke Copilot SDK","centralized":false,"decentralized":false,"label":true,"source_file":"smoke-copilot-sdk"}]' GH_AW_HELP_COMMAND_ENABLED: 'true' GH_AW_SLASH_COMMAND_DOCS_URL: 'https://github.github.com/gh-aw/reference/command-triggers/' with: diff --git a/pkg/workflow/schemas/github-workflow.json b/pkg/workflow/schemas/github-workflow.json index fd902c7129e..d155681f698 100644 --- a/pkg/workflow/schemas/github-workflow.json +++ b/pkg/workflow/schemas/github-workflow.json @@ -260,9 +260,6 @@ "discussions": { "$ref": "#/definitions/permissions-level" }, - "drives": { - "$ref": "#/definitions/permissions-level" - }, "id-token": { "$ref": "#/definitions/permissions-level" }, @@ -295,6 +292,9 @@ "type": "string", "enum": ["write", "none"] }, + "drives": { + "$ref": "#/definitions/permissions-level" + }, "vulnerability-alerts": { "type": "string", "enum": ["read", "none"] From 455639fcfac213cfa0965a2ede15fad7edcc8a29 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:41:49 +0000 Subject: [PATCH 14/18] fix grader sandbox clone fallback and parser edge cases Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 12 +++++++-- actions/setup/js/trace_graders.test.cjs | 11 +++++++- actions/setup/js/trace_graders_worker.cjs | 26 +++++++----------- pkg/workflow/graders_config.go | 11 +++++++- pkg/workflow/graders_config_test.go | 33 +++++++++++++++++++++++ 5 files changed, 72 insertions(+), 21 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 6d39e174eb3..6c42c79abbe 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -320,19 +320,27 @@ const BUILTIN_META = { "artifact-production": { unit: "count", direction: "higher_is_better" }, }; +/** + * @param {any} toolCall + * @returns {boolean} + */ +function isToolFailure(toolCall) { + return toolCall.success === false || toolCall.status === "error" || toolCall.status === "failure" || toolCall.error !== undefined; +} + /** * @param {PreprocessedTrace} trace * @returns {number} Success rate of tool calls (0-1), or 1 if no tool calls */ function gradeToolSuccessRate(trace) { if (trace.toolCalls.length === 0) return 1; - const successes = trace.toolCalls.filter(t => t.success === true || (t.success !== false && t.status !== "error" && t.status !== "failure" && t.error === undefined)).length; + const successes = trace.toolCalls.length - trace.toolCalls.filter(isToolFailure).length; return successes / trace.toolCalls.length; } /** @param {PreprocessedTrace} trace @returns {number} */ function gradeToolFailureCount(trace) { - return trace.toolCalls.filter(t => t.success === false || t.status === "error" || t.status === "failure" || t.error !== undefined).length; + return trace.toolCalls.filter(isToolFailure).length; } /** @param {PreprocessedTrace} trace @returns {number} */ diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index 96bca660a1c..27bd55740db 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -198,6 +198,15 @@ describe("trace_graders", () => { }); expect(gradeToolFailureCount(trace)).toBe(2); }); + + it("matches success-rate complement for ambiguous calls", () => { + const trace = makeTrace({ + toolCalls: [{ name: "a", success: true }, { name: "b", status: "error" }, { name: "c" }], + }); + const failures = gradeToolFailureCount(trace); + const successRate = gradeToolSuccessRate(trace); + expect(successRate).toBeCloseTo((trace.toolCalls.length - failures) / trace.toolCalls.length); + }); }); describe("gradeRetries", () => { @@ -491,7 +500,7 @@ describe("trace_graders", () => { }); it("handles nested malicious JSON in JSONL", () => { - const hostile = '{"a": 1, "__proto__": {"polluted": true}}'; + const hostile = '{"a":1,"constructor":{"prototype":{"polluted":true}}}'; const result = safeParseJsonl(hostile); expect(result.length).toBe(1); expect(result[0].a).toBe(1); diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs index 6eddcbcf577..cce580fd9a0 100644 --- a/actions/setup/js/trace_graders_worker.cjs +++ b/actions/setup/js/trace_graders_worker.cjs @@ -3,25 +3,17 @@ const vm = require("vm"); /** - * @param {any} obj + * @param {any} value + * @param {string} label * @returns {any} */ -function deepClone(obj) { - if (obj === null || obj === undefined) return obj; +function tryStructuredClone(value, label) { + if (value === null || value === undefined) return value; try { - return structuredClone(obj); + return structuredClone(value); } catch { - if (Array.isArray(obj)) { - return obj.map(item => deepClone(item)); - } - if (obj !== null && typeof obj === "object") { - const out = {}; - for (const [k, v] of Object.entries(obj)) { - out[k] = deepClone(v); - } - return out; - } - return obj; + process.stderr.write(`grader worker: failed to structuredClone ${label}; value will default to {}\n`); + return undefined; } } @@ -64,8 +56,8 @@ async function main() { } try { - const trace = deepFreeze(deepClone(payload.trace || {})); - const config = deepFreeze(deepClone(payload.config || {})); + const trace = deepFreeze(tryStructuredClone(payload.trace, "trace") ?? {}); + const config = deepFreeze(tryStructuredClone(payload.config, "config") ?? {}); const run = deepFreeze({ graderCount: Number(payload.graderCount) || 0 }); const workflow = deepFreeze({}); const script = String(payload.script || ""); diff --git a/pkg/workflow/graders_config.go b/pkg/workflow/graders_config.go index f3c64797113..032734adccd 100644 --- a/pkg/workflow/graders_config.go +++ b/pkg/workflow/graders_config.go @@ -206,8 +206,14 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra } // Parse explicit entries + seenNormalizedIDs := make(map[string]string, len(m)) for id, entryRaw := range m { + rawID := id id = strings.TrimSpace(id) + if existingRawID, exists := seenNormalizedIDs[id]; exists { + return nil, fmt.Errorf("graders has duplicate id %q after normalization. Remove whitespace variants (for example %q and %q)", id, existingRawID, rawID) + } + seenNormalizedIDs[id] = rawID if !graderIDPattern.MatchString(id) { return nil, fmt.Errorf("graders has invalid id %q: must match %s. Example:\ngraders:\n my-metric:\n script: \"return { value: trace.toolCalls.length }\"", id, graderIDPattern.String()) } @@ -219,7 +225,11 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra def = builtinDefFromMeta(meta) } + _, isBuiltin := builtinSet[id] if entryRaw == nil { + if !isBuiltin { + return nil, fmt.Errorf("graders.%s is not a built-in grader and requires a 'script' field. Built-in graders: %s", id, strings.Join(BuiltinGraderIDs, ", ")) + } cfg.Graders[id] = def continue } @@ -229,7 +239,6 @@ func (c *Compiler) parseGradersFromFrontmatter(frontmatter map[string]any) (*Gra return nil, fmt.Errorf("graders.%s must be a map or null, got %T. Example:\ngraders:\n %s:\n enabled: true", id, entryRaw, id) } - _, isBuiltin := builtinSet[id] if err := parseGraderEntryFields(def, entry, id, isBuiltin); err != nil { return nil, err } diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go index 23265572cc8..825eb4e817f 100644 --- a/pkg/workflow/graders_config_test.go +++ b/pkg/workflow/graders_config_test.go @@ -160,6 +160,22 @@ func TestParseGradersFromFrontmatter_CustomWithoutScript(t *testing.T) { } } +// TestParseGradersFromFrontmatter_CustomNullRejected verifies null custom graders are rejected at parse time. +func TestParseGradersFromFrontmatter_CustomNullRejected(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "my-custom": nil, + }, + }) + if err == nil { + t.Fatal("expected error for null custom grader") + } + if !strings.Contains(err.Error(), "requires a 'script' field") { + t.Fatalf("expected missing-script error, got: %v", err) + } +} + // TestParseGradersFromFrontmatter_ScriptLengthUsesCharacters verifies limits align to character count. func TestParseGradersFromFrontmatter_ScriptLengthUsesCharacters(t *testing.T) { var c Compiler @@ -199,6 +215,23 @@ func TestParseGradersFromFrontmatter_InvalidID(t *testing.T) { } } +// TestParseGradersFromFrontmatter_DuplicateNormalizedID verifies whitespace variants are rejected. +func TestParseGradersFromFrontmatter_DuplicateNormalizedID(t *testing.T) { + var c Compiler + _, err := c.parseGradersFromFrontmatter(map[string]any{ + "graders": map[string]any{ + "retry-test": map[string]any{"script": "return 1"}, + " retry-test": map[string]any{"script": "return 2"}, + }, + }) + if err == nil { + t.Fatal("expected duplicate normalized id error") + } + if !strings.Contains(err.Error(), "duplicate id") { + t.Fatalf("expected duplicate id error, got: %v", err) + } +} + // TestParseGradersFromFrontmatter_AllDisabledError verifies error when all disabled. func TestParseGradersFromFrontmatter_AllDisabledError(t *testing.T) { var c Compiler From 1d3542e764bb791288772193c7ae7a84000594b5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:49:09 +0000 Subject: [PATCH 15/18] fix: rename grader clone helper to explicit clone-or-undefined Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/trace_graders_worker.cjs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/actions/setup/js/trace_graders_worker.cjs b/actions/setup/js/trace_graders_worker.cjs index cce580fd9a0..53004ef48e8 100644 --- a/actions/setup/js/trace_graders_worker.cjs +++ b/actions/setup/js/trace_graders_worker.cjs @@ -7,8 +7,7 @@ const vm = require("vm"); * @param {string} label * @returns {any} */ -function tryStructuredClone(value, label) { - if (value === null || value === undefined) return value; +function tryStructuredCloneOrUndefined(value, label) { try { return structuredClone(value); } catch { @@ -56,8 +55,8 @@ async function main() { } try { - const trace = deepFreeze(tryStructuredClone(payload.trace, "trace") ?? {}); - const config = deepFreeze(tryStructuredClone(payload.config, "config") ?? {}); + const trace = deepFreeze(tryStructuredCloneOrUndefined(payload.trace, "trace") ?? {}); + const config = deepFreeze(tryStructuredCloneOrUndefined(payload.config, "config") ?? {}); const run = deepFreeze({ graderCount: Number(payload.graderCount) || 0 }); const workflow = deepFreeze({}); const script = String(payload.script || ""); From 6c61d489a0e1884296d8c23e967c20ed90326a68 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:13:10 +0000 Subject: [PATCH 16/18] refactor: rename trace graders wording to graders Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 2 +- actions/setup/sh/prepare_threat_detection_files.sh | 2 +- docs/src/content/docs/reference/trace-graders.md | 6 +++--- pkg/parser/schemas/main_workflow_schema.json | 2 +- pkg/workflow/compiler_orchestrator_workflow.go | 2 +- pkg/workflow/compiler_yaml_graders.go | 6 +++--- pkg/workflow/compiler_yaml_post_agent.go | 2 +- pkg/workflow/frontmatter_types.go | 2 +- pkg/workflow/graders_config.go | 4 ++-- pkg/workflow/graders_config_test.go | 6 +++--- pkg/workflow/graders_workflow_integration_test.go | 2 +- pkg/workflow/workflow_data.go | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 6c42c79abbe..157eca8fea6 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -737,7 +737,7 @@ async function main(manifestB64, execSpecB64) { } // Step summary - core.summary.addHeading("Trace Graders", 3); + core.summary.addHeading("Graders", 3); const tableResults = results.filter(r => r.status !== "unavailable"); if (tableResults.length > 0) { const rows = tableResults.map(r => { diff --git a/actions/setup/sh/prepare_threat_detection_files.sh b/actions/setup/sh/prepare_threat_detection_files.sh index b676b6974cb..d348835d5ed 100755 --- a/actions/setup/sh/prepare_threat_detection_files.sh +++ b/actions/setup/sh/prepare_threat_detection_files.sh @@ -47,7 +47,7 @@ for artifact_pattern in aw-*.patch aw-*.bundle; do done done -# Copy grader manifest and results if present (deterministic trace graders) +# Copy grader manifest and results if present (deterministic graders) GRADER_SOURCE_DIR="${SOURCE_DIR}/agent/graders" if [ -d "${GRADER_SOURCE_DIR}" ]; then GRADER_DETECTION_DIR="${DETECTION_DIR}/agent/graders" diff --git a/docs/src/content/docs/reference/trace-graders.md b/docs/src/content/docs/reference/trace-graders.md index 11b802f42c9..3be39b12628 100644 --- a/docs/src/content/docs/reference/trace-graders.md +++ b/docs/src/content/docs/reference/trace-graders.md @@ -1,12 +1,12 @@ --- -title: Trace Graders +title: Graders description: Deterministic metrics computed from agent execution traces --- -Trace graders compute deterministic metrics from post-agent execution trace files (token usage, MCP gateway logs, agent output) without LLM calls or network access. Results are persisted in the agent artifact for downstream consumption by detection jobs and reporting tools. +Graders compute deterministic metrics from post-agent execution trace files (token usage, MCP gateway logs, agent output) without LLM calls or network access. Results are persisted in the agent artifact for downstream consumption by detection jobs and reporting tools. :::caution[Experimental] -Trace graders are an experimental feature. +Graders are an experimental feature. ::: ## Quick start diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 4acdd73aa4d..73b9eb742e4 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -12743,7 +12743,7 @@ "examples": ["gpt-5.4", "claude-3-5-sonnet-20241022", "gpt-4"] }, "graders": { - "description": "\u26a0\ufe0f Experimental. Deterministic trace graders to compute post-agent metrics from execution artifacts. Map keys are grader IDs. Built-in graders can be configured by ID; custom graders require a script.", + "description": "\u26a0\ufe0f Experimental. Deterministic graders to compute post-agent metrics from execution artifacts. Map keys are grader IDs. Built-in graders can be configured by ID; custom graders require a script.", "type": "object", "propertyNames": { "pattern": "^[a-z][a-z0-9-]{0,63}$", diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 69a279b0fe4..2b1f234df15 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -568,7 +568,7 @@ func (c *Compiler) extractAdditionalConfigurations( } workflowData.Evals = evalsConfig - // Extract deterministic trace graders configuration. + // Extract deterministic graders configuration. gradersConfig, err := c.parseGradersFromFrontmatter(frontmatter) if err != nil { return fmt.Errorf("invalid graders configuration: %w", err) diff --git a/pkg/workflow/compiler_yaml_graders.go b/pkg/workflow/compiler_yaml_graders.go index d029dcd93aa..9002fcffae4 100644 --- a/pkg/workflow/compiler_yaml_graders.go +++ b/pkg/workflow/compiler_yaml_graders.go @@ -14,7 +14,7 @@ import ( var compilerYamlGradersLog = logger.New("workflow:compiler_yaml_graders") // generateGradersStep emits an always() post-agent step that runs deterministic -// trace graders. The step executes after secret redaction / summary steps and before +// graders. The step executes after secret redaction / summary steps and before // the unified artifact upload so results are included in the agent artifact. // // The step is only emitted when graders are configured (graders: in frontmatter). @@ -43,7 +43,7 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData } execB64 := base64.StdEncoding.EncodeToString(execJSON) - yaml.WriteString(" - name: Run trace graders\n") + yaml.WriteString(" - name: Run graders\n") yaml.WriteString(" if: always()\n") yaml.WriteString(" continue-on-error: true\n") fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) @@ -54,7 +54,7 @@ func (c *Compiler) generateGradersStep(yaml *strings.Builder, data *WorkflowData yaml.WriteString(" const { main } = require('" + SetupActionDestination + "/trace_graders.cjs');\n") fmt.Fprintf(yaml, " await main('%s', '%s');\n", manifestB64, execB64) - compilerYamlGradersLog.Print("Generated trace graders step") + compilerYamlGradersLog.Print("Generated graders step") } // graderManifestEntry represents a single grader in the serialized manifest. diff --git a/pkg/workflow/compiler_yaml_post_agent.go b/pkg/workflow/compiler_yaml_post_agent.go index f26d02e7474..93ed5dfeef2 100644 --- a/pkg/workflow/compiler_yaml_post_agent.go +++ b/pkg/workflow/compiler_yaml_post_agent.go @@ -197,7 +197,7 @@ func (c *Compiler) generatePostAgentCollectionAndUpload(yaml *strings.Builder, d // Emit all GITHUB_STEP_SUMMARY log-parsing steps. c.generateSummarySteps(yaml, data, engine) - // Run deterministic trace graders after trace data is available. + // Run deterministic graders after trace data is available. c.generateGradersStep(yaml, data) // Re-scan grader output files for leaked secrets when custom grader scripts diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go index 83ad1ffd464..52ff34a5808 100644 --- a/pkg/workflow/frontmatter_types.go +++ b/pkg/workflow/frontmatter_types.go @@ -452,7 +452,7 @@ type FrontmatterConfig struct { // engine-config / runs-on overrides. Evals any `json:"evals,omitempty"` - // Graders configures deterministic trace graders that compute metrics from + // Graders configures deterministic graders that compute metrics from // post-agent trace files. Can be {} for zero-config (all built-ins) or a map // of grader IDs with optional enabled/script overrides. Graders any `json:"graders,omitempty"` diff --git a/pkg/workflow/graders_config.go b/pkg/workflow/graders_config.go index 032734adccd..4c3eb2444a3 100644 --- a/pkg/workflow/graders_config.go +++ b/pkg/workflow/graders_config.go @@ -1,4 +1,4 @@ -// Package workflow - Deterministic trace graders configuration types and parser. +// Package workflow - Deterministic graders configuration types and parser. package workflow import ( @@ -83,7 +83,7 @@ func (g *GraderDefinition) ScriptDigest() string { return hex.EncodeToString(h[:]) } -// GradersConfig holds the configuration for deterministic trace graders declared +// GradersConfig holds the configuration for deterministic graders declared // in workflow frontmatter. Graders run as an always() post-agent step in the agent job. type GradersConfig struct { // Graders is the map of grader ID to definition. diff --git a/pkg/workflow/graders_config_test.go b/pkg/workflow/graders_config_test.go index 825eb4e817f..ace2ab91d4e 100644 --- a/pkg/workflow/graders_config_test.go +++ b/pkg/workflow/graders_config_test.go @@ -335,8 +335,8 @@ func TestGenerateGradersStep_Present(t *testing.T) { } c.generateGradersStep(&yaml, data) output := yaml.String() - if !strings.Contains(output, "Run trace graders") { - t.Fatal("expected step name 'Run trace graders'") + if !strings.Contains(output, "Run graders") { + t.Fatal("expected step name 'Run graders'") } if !strings.Contains(output, "if: always()") { t.Fatal("expected always() condition") @@ -374,7 +374,7 @@ func TestGenerateGradersStep_BeforeArtifactUpload(t *testing.T) { yaml.WriteString(" - name: Upload agent artifacts\n") output := yaml.String() - graderIdx := strings.Index(output, "Run trace graders") + graderIdx := strings.Index(output, "Run graders") uploadIdx := strings.Index(output, "Upload agent artifacts") if graderIdx < 0 || uploadIdx < 0 { t.Fatal("expected both steps to be present") diff --git a/pkg/workflow/graders_workflow_integration_test.go b/pkg/workflow/graders_workflow_integration_test.go index 58dc4ba9f9c..a4ed7b0ae59 100644 --- a/pkg/workflow/graders_workflow_integration_test.go +++ b/pkg/workflow/graders_workflow_integration_test.go @@ -62,7 +62,7 @@ experiments: require.NoError(t, err) yaml := string(compiled) - assert.Contains(t, yaml, "Run trace graders") + assert.Contains(t, yaml, "Run graders") assert.Contains(t, yaml, "trace_graders.cjs") assert.Contains(t, yaml, "grader:retries") assert.Contains(t, yaml, "graders.custom-score.value") diff --git a/pkg/workflow/workflow_data.go b/pkg/workflow/workflow_data.go index 80f93a2a29a..f033f4d2cba 100644 --- a/pkg/workflow/workflow_data.go +++ b/pkg/workflow/workflow_data.go @@ -208,7 +208,7 @@ type WorkflowData struct { ContainerPinMappings map[string]string // container-pin redirect table from aw.json container_pins: maps source image → replacement image GHES bool // select action versions compatible with GitHub Enterprise Server Evals *EvalsConfig // BinEval evaluation configuration parsed from frontmatter evals field - Graders *GradersConfig // Deterministic trace graders configuration parsed from frontmatter graders field + Graders *GradersConfig // Deterministic graders configuration parsed from frontmatter graders field ExcludedEnv []string // additional env var names to exclude from agent container via AWF --exclude-env (from frontmatter excluded-env field) } From 5131cd971505b0656fd5a0a50e9c3afae1903f7e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:29:04 +0000 Subject: [PATCH 17/18] refactor: rename grader implementation id Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/trace_graders.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 157eca8fea6..b58bd98f6fd 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -35,7 +35,7 @@ const SCRIPT_WORKER_OVERHEAD_MS = 1000; // Allow worker startup/serialization ov const SCRIPT_WORKER_PATH = path.join(__dirname, "trace_graders_worker.cjs"); const GRADER_VERSION = 1; -const IMPLEMENTATION_ID = "gh-aw/trace-graders"; +const IMPLEMENTATION_ID = "gh-aw/graders"; // --- Trace preprocessing --- From 782f56dacff6e6d23fb39844041c95be6b6ec2b2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:41:58 +0000 Subject: [PATCH 18/18] docs: add graders specification Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../content/docs/reference/trace-graders.md | 2 + .../docs/specs/graders-specification.md | 290 ++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 docs/src/content/docs/specs/graders-specification.md diff --git a/docs/src/content/docs/reference/trace-graders.md b/docs/src/content/docs/reference/trace-graders.md index 3be39b12628..77068a9b7a6 100644 --- a/docs/src/content/docs/reference/trace-graders.md +++ b/docs/src/content/docs/reference/trace-graders.md @@ -5,6 +5,8 @@ description: Deterministic metrics computed from agent execution traces Graders compute deterministic metrics from post-agent execution trace files (token usage, MCP gateway logs, agent output) without LLM calls or network access. Results are persisted in the agent artifact for downstream consumption by detection jobs and reporting tools. +For normative requirements, see the [Graders Specification](/gh-aw/specs/graders-specification/). + :::caution[Experimental] Graders are an experimental feature. ::: diff --git a/docs/src/content/docs/specs/graders-specification.md b/docs/src/content/docs/specs/graders-specification.md new file mode 100644 index 00000000000..06c74f4b8cb --- /dev/null +++ b/docs/src/content/docs/specs/graders-specification.md @@ -0,0 +1,290 @@ +--- +title: Graders Specification +description: Formal specification for deterministic gh-aw graders and grader-backed experiment metrics +sidebar: + order: 1360 +--- + +# Graders Specification + +**Version**: 0.1.0 +**Status**: Draft Specification +**Feature Status**: Experimental +**Latest Version**: [graders-specification](/gh-aw/specs/graders-specification/) +**Editor**: GitHub Agentic Workflows Team + +--- + +## Abstract + +This specification defines the `graders` feature in gh-aw: deterministic, post-agent metrics computed from execution traces and persisted as structured artifacts. It specifies configuration, built-in grader behavior, custom inline grader constraints, execution ordering, artifact outputs, experiment metric references, and conformance requirements. + +## Status of This Document + +This section describes the status of this document at the time of publication. This is a draft specification and may be updated, replaced, or made obsolete by other documents at any time. + +This feature is experimental and implementations SHOULD expect iteration before final recommendation status. + +## Table of Contents + +1. [Introduction](#1-introduction) +2. [Conformance](#2-conformance) +3. [Architecture](#3-architecture) +4. [Configuration Model](#4-configuration-model) +5. [Built-in Graders](#5-built-in-graders) +6. [Custom Inline Graders](#6-custom-inline-graders) +7. [Execution and Artifacts](#7-execution-and-artifacts) +8. [Experiment Metric References](#8-experiment-metric-references) +9. [Security and Isolation](#9-security-and-isolation) +10. [Compliance Testing](#10-compliance-testing) +11. [Norms](#11-norms) +12. [References](#12-references) +13. [Change Log](#13-change-log) + +--- + +## 1. Introduction + +### 1.1 Purpose + +The `graders` feature provides deterministic quality and behavior metrics derived from workflow trace artifacts without issuing additional LLM calls. + +### 1.2 Scope + +This specification covers: + +- Frontmatter configuration under `graders` +- Built-in grader identifiers and semantics +- Custom inline grader script requirements +- Output artifact contracts +- Experiment metric integration for grader references + +This specification does NOT cover: + +- Non-deterministic evaluator models +- UI visualization requirements +- External metric backends + +### 1.3 Design Goals + +A conforming implementation: + +1. MUST compute grader values deterministically from run artifacts. +2. MUST preserve stable grader IDs for experiment references. +3. SHOULD keep grading isolated from network-dependent behavior. +4. MUST emit machine-readable grader artifacts for downstream tooling. + +--- + +## 2. Conformance + +### 2.1 Conformance Classes + +- **Conforming implementation**: Satisfies all MUST/SHALL requirements in this document. +- **Partially conforming implementation**: Supports built-in graders but omits custom inline grader execution. + +### 2.2 Requirements Notation + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +### 2.3 Compliance Levels + +- **Level 1 (Required)**: Built-in graders, manifest/results output. +- **Level 2 (Standard)**: Custom inline graders with validation and isolation. +- **Level 3 (Complete)**: Experiment metric references to graders with validation. + +--- + +## 3. Architecture + +`graders` executes as a post-agent step in the existing `agent` job: + +1. Parse and validate frontmatter `graders`. +2. Build grader manifest and execution spec. +3. Preprocess trace artifacts once. +4. Execute enabled graders (built-in and custom inline). +5. Write normalized outputs to grader artifact files. + +The grading step MUST run with `if: always()` semantics and SHOULD continue even when individual graders fail, recording per-grader errors in results. + +--- + +## 4. Configuration Model + +### 4.1 Frontmatter Key + +The configuration key MUST be `graders`. + +### 4.2 Enable/Disable Semantics + +- If `graders` is omitted, grading MUST be disabled. +- If `graders: {}` is provided, all built-in graders MUST be enabled with defaults. +- If `graders` is present, at least one grader MUST be enabled; otherwise configuration MUST fail. + +### 4.3 Entry Model + +`graders` is a map of ` -> `. + +- Built-in grader entries MAY be `null` to enable defaults. +- Custom grader entries MUST be objects and MUST include `script`. + +Supported object fields include `enabled`, `name`, `description`, `unit`, `direction`, `threshold`, `min`, `max`, `config`, and `script`. + +--- + +## 5. Built-in Graders + +The implementation MUST recognize the following built-in grader IDs: + +- `tool-success-rate` +- `tool-failure-count` +- `retries` +- `loops` +- `trajectory-efficiency` +- `execution-step-count` +- `execution-duration` +- `context-growth` +- `artifact-production` + +These IDs are reserved for built-ins. A built-in grader MUST NOT accept a custom `script`. + +--- + +## 6. Custom Inline Graders + +A custom grader is any grader ID not in the built-in set. + +### 6.1 Required Fields + +A custom grader MUST define `script`. + +### 6.2 Script Limits + +- `script` MUST be non-empty. +- `script` MUST NOT exceed 4096 characters. + +### 6.3 Forbidden Patterns + +Inline scripts MUST be rejected if they contain any forbidden pattern, including: + +- `require(` +- `import(` +- `import ` +- `fetch(` +- `eval(` +- `process.exit` +- `child_process` +- `execSync` +- `spawnSync` +- `Function(` + +--- + +## 7. Execution and Artifacts + +### 7.1 Output Directory + +Graders output MUST be written under: + +`/tmp/gh-aw/agent/graders` + +### 7.2 Required Files + +The implementation MUST produce: + +- `grader_manifest.json` +- `grader_results.json` + +### 7.3 Artifact Inclusion + +Both files MUST be included in the unified `agent` artifact. + +### 7.4 Deterministic Output Contract + +`grader_results.json` SHOULD include normalized run/result structures suitable for downstream programmatic reads, including per-grader value/status and run-level pass/fail/error counts. + +--- + +## 8. Experiment Metric References + +Experiment metric fields MAY reference grader outputs. + +Supported forms include: + +- `grader:` +- `graders..value` + +When a grader reference is used, `` MUST resolve to a declared enabled grader. Unknown or empty grader references MUST fail validation. + +--- + +## 9. Security and Isolation + +- Grading MUST operate on local run artifacts and MUST NOT require outbound network access for built-ins. +- Custom inline graders MUST execute in a restricted context with blocked dangerous primitives. +- Implementations SHOULD enforce bounded execution time for inline scripts. +- Implementations SHOULD redact grader outputs when custom scripts are enabled to reduce secret leakage risk. + +--- + +## 10. Compliance Testing + +### 10.1 Test Suite Requirements + +- **T-GRD-001**: Omitted `graders` key disables grading step emission. +- **T-GRD-002**: `graders: {}` enables all built-ins. +- **T-GRD-003**: Unknown custom grader without `script` is rejected. +- **T-GRD-004**: Custom `script` over 4096 chars is rejected. +- **T-GRD-005**: Forbidden script patterns are rejected. +- **T-GRD-006**: Built-in grader with `script` is rejected. +- **T-GRD-007**: `grader_manifest.json` is written to required path. +- **T-GRD-008**: `grader_results.json` is written to required path. +- **T-GRD-009**: Grader files are present in `agent` artifact. +- **T-GRD-010**: `experiments.*.metric` with `grader:` validates declared enabled grader. +- **T-GRD-011**: `experiments.*.metric` with `graders..value` validates declared enabled grader. + +### 10.2 Compliance Checklist + +| Requirement | Test ID | Level | Status | +|---|---|---|---| +| Frontmatter key is `graders` | T-GRD-001 | 1 | Required | +| Empty map enables built-ins | T-GRD-002 | 1 | Required | +| Custom graders require script | T-GRD-003 | 2 | Required | +| Script safety constraints enforced | T-GRD-004, T-GRD-005 | 2 | Required | +| Required artifact files emitted | T-GRD-007, T-GRD-008 | 1 | Required | +| Experiment grader references validate | T-GRD-010, T-GRD-011 | 3 | Required | + +--- + +## 11. Norms + +- **N-GRD-001**: Implementations MUST treat `graders` as experimental. +- **N-GRD-002**: Implementations MUST preserve built-in grader ID stability across patch releases. +- **N-GRD-003**: Implementations SHOULD preserve deterministic output for identical trace inputs. +- **N-GRD-004**: Implementations MUST fail fast on invalid custom grader scripts. +- **N-GRD-005**: Implementations MUST keep grader artifact paths stable unless a major version change is issued. + +--- + +## 12. References + +### Normative References + +- **[RFC 2119]** Key words for use in RFCs to Indicate Requirement Levels. + https://www.ietf.org/rfc/rfc2119.txt + +### Informative References + +- **[Graders Reference]** [Graders](/gh-aw/reference/trace-graders/) +- **[Experiments Specification]** [Experiments Specification](/gh-aw/experimental/experiments-specification/) + +--- + +## 13. Change Log + +### Version 0.1.0 (Draft Specification) + +- Initial draft for gh-aw graders. +- Defines `graders` configuration semantics and built-in grader set. +- Defines custom inline grader constraints and forbidden patterns. +- Defines grader artifact output contract and experiment metric references.