Add initial deterministic trace grading framework

actions/setup/js/experiment_helpers.cjs

@@ -36,8 +36,25 @@ function readExperimentAssignments() {
     }
     return null;
   } catch {
-    return null;
+    // Fall through to environment-variable fallback.
+  }
+
+  /** @type {Record<string, string>} */
+  const fromEnv = {};
+  for (const [key, value] of Object.entries(process.env)) {
+    if (!key.startsWith("GH_AW_EXPERIMENTS_")) {
+      continue;
+    }
+    const name = key.substring("GH_AW_EXPERIMENTS_".length).toLowerCase();
+    if (!name || typeof value !== "string" || value.length === 0) {
+      continue;
+    }
+    fromEnv[name] = value;
+  }
+  if (Object.keys(fromEnv).length > 0) {
+    return fromEnv;
   }
+  return null;
 }
 
 module.exports = { readExperimentAssignments, EXPERIMENT_ASSIGNMENTS_PATH };

actions/setup/js/experiment_helpers.test.cjs

@@ -16,6 +16,8 @@ describe("readExperimentAssignments", () => {
 
   afterEach(() => {
     readFileSpy.mockRestore();
+    delete process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE;
+    delete process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH;
     if (savedStateDir !== undefined) {
       process.env.GH_AW_EXPERIMENT_STATE_DIR = savedStateDir;
     } else {
@@ -67,4 +69,10 @@ describe("readExperimentAssignments", () => {
     });
     expect(readExperimentAssignments()).toEqual({ mode: "fast" });
   });
+
+  it("falls back to GH_AW_EXPERIMENTS_* environment variables when assignments file is unavailable", () => {
+    process.env.GH_AW_EXPERIMENTS_PROMPT_STYLE = "concise";
+    process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH = "deep";
+    expect(readExperimentAssignments()).toEqual({ prompt_style: "concise", reasoning_depth: "deep" });
+  });
 });

actions/setup/js/redact_secrets.cjs

@@ -343,4 +343,40 @@ async function main() {
   }
 }
 
-module.exports = { main, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS };
+/**
+ * Targeted redaction pass for files in a specific directory. Used by the
+ * post-graders redaction step to scan grader output files that were written
+ * after the initial full-workspace redaction pass.
+ *
+ * @param {string} dir - Absolute directory path to scan
+ */
+async function redactFilesInDir(dir) {
+  try {
+    const secretNames = (process.env.GH_AW_SECRET_NAMES || "").split(",").filter(n => n.trim());
+    /** @type {string[]} */
+    const secretValues = [];
+    for (const secretName of secretNames) {
+      const value = process.env[`SECRET_${secretName.trim()}`];
+      if (typeof value === "string" && value.trim() !== "") {
+        secretValues.push(value.trim());
+      }
+    }
+    secretValues.push(...extractMCPGatewayTokens(MCP_GATEWAY_CONFIG_PATHS));
+
+    const targetExtensions = [".json"];
+    const files = findFiles(dir, targetExtensions);
+    if (files.length === 0) return;
+
+    let totalRedactions = 0;
+    for (const file of files) {
+      totalRedactions += processFile(file, secretValues);
+    }
+    if (totalRedactions > 0) {
+      core.info(`Grader output redaction: ${totalRedactions} redaction(s) in ${dir}`);
+    }
+  } catch (error) {
+    core.warning(`Grader output redaction failed: ${getErrorMessage(error)}`);
+  }
+}
+
+module.exports = { main, redactFilesInDir, redactSecrets, redactBuiltInPatterns, redactStepSummaryContent, extractMCPGatewayTokens, BUILT_IN_PATTERNS, MCP_GATEWAY_CONFIG_PATHS };

actions/setup/js/run_evals.cjs

@@ -36,6 +36,7 @@ const path = require("path");
 const { ERR_VALIDATION, ERR_SYSTEM } = require("./error_codes.cjs");
 const { getErrorMessage } = require("./error_helpers.cjs");
 const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs");
+const { readExperimentAssignments } = require("./experiment_helpers.cjs");
 const { resolveModelWithFallback } = require("./model_fallback.cjs");
 
 const EVALS_DIR = "/tmp/gh-aw/evals";
@@ -127,6 +128,7 @@ async function parseMain() {
   const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS;
   const model = resolveModelWithFallback(process.env, "GH_AW_EVALS_MODEL") || "";
   const runID = process.env.GITHUB_RUN_ID || "unknown";
+  const experimentAssignments = readExperimentAssignments();
 
   /** @type {Array<{id: string, question: string}>} */
   let questions = [];
@@ -185,6 +187,7 @@ async function parseMain() {
       model,
       timestamp,
       runid: runID,
+      experiments: experimentAssignments || undefined,
     };
     results.push(record);
     core.info(`Q[${q.id}]: ${answer}`);

actions/setup/js/run_evals.test.cjs

@@ -5,6 +5,7 @@ import { createRequire } from "module";
 const EVALS_DIR = "/tmp/gh-aw/evals";
 const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`;
 const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl";
+const EXPERIMENT_ASSIGNMENTS_PATH = "/tmp/gh-aw/experiments/assignments.json";
 const require = createRequire(import.meta.url);
 const { MODEL_FALLBACK_ENV_VAR } = require("./model_fallback.cjs");
 const { setupMain, parseMain, extractAssistantTextFromJsonlLog } = require("./run_evals.cjs");
@@ -32,6 +33,9 @@ describe("run_evals.cjs", () => {
     if (fs.existsSync(EVALS_OUTPUT_PATH)) {
       fs.unlinkSync(EVALS_OUTPUT_PATH);
     }
+    if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) {
+      fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH);
+    }
   });
 
   afterEach(() => {
@@ -42,6 +46,9 @@ describe("run_evals.cjs", () => {
     if (fs.existsSync(EVALS_OUTPUT_PATH)) {
       fs.unlinkSync(EVALS_OUTPUT_PATH);
     }
+    if (fs.existsSync(EXPERIMENT_ASSIGNMENTS_PATH)) {
+      fs.unlinkSync(EXPERIMENT_ASSIGNMENTS_PATH);
+    }
   });
 
   it("stores the workflow run id when writing eval records", async () => {
@@ -64,6 +71,20 @@ describe("run_evals.cjs", () => {
     });
   });
 
+  it("includes experiment assignments in eval records when available", async () => {
+    vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
+    vi.stubEnv("GH_AW_EVALS_MODEL", "small");
+    vi.stubEnv("GITHUB_RUN_ID", "123456789");
+    fs.mkdirSync("/tmp/gh-aw/experiments", { recursive: true });
+    fs.writeFileSync("/tmp/gh-aw/experiments/assignments.json", JSON.stringify({ prompt_style: "concise" }) + "\n", "utf8");
+    fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8");
+
+    await parseMain();
+
+    const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n");
+    expect(JSON.parse(line).experiments).toEqual({ prompt_style: "concise" });
+  });
+
   it('falls back to "unknown" when the workflow run id is absent', async () => {
     vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
     vi.stubEnv("GH_AW_EVALS_MODEL", "small");