diff --git a/.env.example b/.env.example index ba95476..2cec708 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,3 @@ -# Optional, both services: telemetry only. `npm run dev:postgres` starts this database. -DATABASE_URL=postgresql://postgres:postgres@localhost:55432/software_factory - # Both services: Render API key (the gateway starts tasks; workflows manage sandboxes and cancel runs). RENDER_API_KEY= diff --git a/AGENTS.md b/AGENTS.md index b94525e..a630611 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,14 +19,13 @@ This file applies to the whole repository. Read ## What this is -software-factory turns a labeled GitHub issue into a spec, a draft PR, and a COMMENT review. Three claiming tasks (Triage, Spec-to-PR, PR Reviewer) run on Render Workflows, each holding a claim on its issue or PR while it works. Agents run in Render Sandboxes restored from a per-repository snapshot. The status comment on the issue or PR names the run that holds the claim. Postgres is optional and keeps an append-only `events` table for telemetry only. +software-factory turns a labeled GitHub issue into a spec, a draft PR, and a COMMENT review. Three claiming tasks (Triage, Spec-to-PR, PR Reviewer) run on Render Workflows, each holding a claim on its issue or PR while it works. Agents run in Render Sandboxes restored from a per-repository snapshot. The status comment on the issue or PR names the run that holds the claim. There is no database: GitHub and Workflows hold all state, and run telemetry is logged as JSON lines. Two processes deploy: - `app/entrypoints/gateway.ts`: the gateway web service. Verifies webhooks, routes each to one task, and calls `startTask` with the delivery ID as the idempotency key. - `app/entrypoints/workflows.ts`: the Workflows service. Importing `app/workflows/index.ts` registers every task. -`app/sql/` holds the migrations, applied by `npm run db:migrate` as the gateway's pre-deploy command. ## Module map @@ -45,10 +44,9 @@ app/ changeset/ Collects, validates, and publishes sandbox changes through the Git Data API github/ REST client, App JWT and installation tokens, Git Data API, hidden comment markers format/ Renders specs, questions, PR bodies, and reviews as GitHub markdown; sanitizes model text - events/ Optional best-effort emit() over the events table - sql/ Migrations + events/ emit(): run.finished and agent.finished as JSON lines on stdout sandbox/ Shell scripts copied into the snapshot (run-agent, as-runner, verify, collect-changes, ...) -scripts/ setup-app, setup-github, doctor, migrate +scripts/ setup-app, setup-github, doctor tests/ Vitest unit tests; fakes in tests/helpers/ ``` @@ -86,7 +84,7 @@ Import another module through its `index.ts` barrel where one exists. A barrel e Outcome names are snake_case. - Quote every shell argument you build with `shq` (`app/sandbox/shell.ts`). Anything long or security-sensitive that runs in a sandbox belongs in a script in `sandbox/`, which CI checks with shellcheck. - Model text that reaches GitHub goes through the helpers in `app/format/`. -- Task implementations take GitHub, sandboxes, `runAgent`, and run control as injectable dependencies. Tests never call live Render, GitHub, Postgres, or model APIs. +- Task implementations take GitHub, sandboxes, `runAgent`, and run control as injectable dependencies. Tests never call live Render, GitHub, or model APIs. - Keep the suite focused; it currently has 100+ tests. Test durable behavior at process and trust boundaries: authenticated inputs, user-visible outcomes, authority, cleanup, and security invariants. Prefer one end-to-end happy path and the @@ -116,7 +114,6 @@ each is enforced): 8. Automation ends at a draft PR and a COMMENT review. It never approves, requests changes, marks a PR ready, or merges. 9. Spec-to-PR implements the spec as it stood when the approval label was applied. 10. Every sandbox is created with a timeout and closed by `withSandbox` (in `finally`). A claiming task records its sandbox in the status comment, so the run or `/factory cancel` that stops it can terminate the sandbox: Render kills a cancelled task without running `finally`. -11. Postgres is telemetry only; no task behavior depends on it. `tests/agents-catalog.test.ts` protects unique agent identities and tool authority. Change it only on purpose. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5f4ba7..9fde781 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Sandbox boundaries, credentials, or publishing. ## Local development -Local development requires Docker and Render CLI 2.29 or later. It still uses +Local development requires Render CLI 2.29 or later. It still uses real Render Sandboxes, a real GitHub App, and a real model. Use a separate test repository and preferably a separate App. @@ -18,15 +18,9 @@ Fill `.env`, then start each process in its own terminal: ```bash # Terminal 1 -npm run dev:postgres - -# Terminal 2 -npm run db:migrate - -# Terminal 3 npm run dev:workflows -# Terminal 4 +# Terminal 2 RENDER_USE_LOCAL_DEV=true npm run dev:gateway ``` @@ -58,8 +52,8 @@ Task input schemas are in npm run check ``` -This runs Biome, TypeScript, and Vitest. Tests fake GitHub, Render, Postgres, -and model calls. +This runs Biome, TypeScript, and Vitest. Tests fake GitHub, Render, and model +calls. When relevant, also run: @@ -173,6 +167,21 @@ Never accept write destinations from model output. Every GitHub write should update or reuse an App-authored artifact marker. Workflows idempotency alone has a finite scope and is not sufficient. +## Add persistent state + +The factory keeps no database: GitHub and Render Workflows hold the state it +needs, and telemetry goes to logs. When a feature needs its own data, such as +a spec library, budgets, or run history you query: + +1. Add a Render Postgres database to `render.yaml` and wire `DATABASE_URL` + into the services that need it with `fromDatabase`. +2. Run migrations from the gateway's `preDeployCommand`, so each deploy + migrates before new code starts. +3. Use the internal connection string on Render. The external one is for + local tools and requires TLS. +4. Keep run coordination in the status comment. A database that also decides + who owns an issue becomes a second source of truth to reconcile. + ## Code structure ```text @@ -186,10 +195,10 @@ app/ sandbox/ Sandbox and snapshot lifecycle changeset/ Candidate-tree validation and publishing github/ App authentication, REST, Git Data, and markers - events/ Optional best-effort telemetry + events/ Run and agent telemetry as JSON log lines schemas/ Cross-process Zod contracts sandbox/ Runtime scripts copied into snapshots -scripts/ Setup, doctor, and migrations +scripts/ Setup and doctor tests/ Unit and boundary tests with fakes ``` diff --git a/README.md b/README.md index 0d95956..2b26878 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ a verified draft pull request, and an automated code review. People approve every transition; the factory never approves or merges changes. It runs in your Render workspace using one web service, one Workflows -service, Postgres, and Render Sandboxes. +service, and Render Sandboxes. **Start by [forking this repository](https://github.com/render-examples/software-factory/fork).** The factory must deploy from your fork because `factory.config.ts` defines @@ -28,7 +28,7 @@ pull request --factory:pr-review--> COMMENT review GitHub remains where people discuss and approve work. Render Workflows owns durable execution. Sandboxes contain model-directed tools and repository -code. Postgres is optional and only records telemetry. +code. ## Deploy diff --git a/app/agents/run.ts b/app/agents/run.ts index 5365a20..bcdf98f 100644 --- a/app/agents/run.ts +++ b/app/agents/run.ts @@ -57,21 +57,25 @@ export async function runAgent) => - void emit({ traceId: base.traceId, runId: base.runId, repo: base.repo, subject: base.subject, type, payload }); const finish = (outcome: string, summary: Summary, message?: string) => - record("agent.finished", { - agentId: agent.id, - model: `${provider}/${model}`, - outcome, - turns: summary.turns, - usage: summary.usage, - costUsd: summary.costUsd, - ...(summary.invalidLines ? { invalidLines: summary.invalidLines } : {}), - ...(message ? { message } : {}), + emit({ + traceId: base.traceId, + runId: base.runId, + repo: base.repo, + subject: base.subject, + type: "agent.finished", + payload: { + agentId: agent.id, + model: `${provider}/${model}`, + outcome, + turns: summary.turns, + usage: summary.usage, + costUsd: summary.costUsd, + ...(summary.invalidLines ? { invalidLines: summary.invalidLines } : {}), + ...(message ? { message } : {}), + }, }); - record("agent.started", { agentId: agent.id, model: `${provider}/${model}`, tools: agent.tools }); let summary = emptySummary(); const failure = (code: AgentErrorCode, text: string) => new AgentError(agent.id, code, text); try { diff --git a/app/claims/claim.ts b/app/claims/claim.ts index 3a1cd73..35244dc 100644 --- a/app/claims/claim.ts +++ b/app/claims/claim.ts @@ -85,7 +85,6 @@ export async function withClaim( work: (claim: Claim) => Promise, ): Promise { const run = new ClaimedRun(ctx, opts); - await run.recordStart(); try { if (!(await run.authorized())) return await run.reject(); await run.acquire(); @@ -149,10 +148,6 @@ class ClaimedRun implements Claim { return { sandbox, ...this.#event }; } - async recordStart(): Promise { - await emit({ ...this.#event, type: "run.started", payload: { stage: this.#opts.task, actor: this.#opts.actor } }); - } - authorized(): Promise { return checkPermission(this.gh, this.#opts.actor); } @@ -161,20 +156,20 @@ class ClaimedRun implements Claim { async reject(): Promise { const summary = `@${this.#opts.actor} needs \`${config.minimumPermission}\` permission on this repository to start it.`; await finishStatus(this.#status("rejected", summary)); - return await this.#record("rejected", "permission_denied", summary); + return this.#record("rejected", "permission_denied", summary); } /** Stops the previous run and rewrites the status comment to name this one, then confirms no other run's write landed after it. */ async acquire(): Promise { const { task, title, runs } = this.#opts; - await takeOver({ gh: this.gh, subject: this.subject, task, title, runId: this.runId, runs }); + await takeOver({ gh: this.gh, repo: this.repo.fullName, subject: this.subject, task, title, runId: this.runId, runs }); await this.checkpoint(); } async finish(outcome: Outcome): Promise { const state = outcome.state ?? "done"; if (!(await this.#publish(state, outcome.summary, outcome.links))) throw new Superseded(); - return await this.#record(state, outcome.outcome, outcome.summary); + return this.#record(state, outcome.outcome, outcome.summary); } /** @@ -182,21 +177,21 @@ class ClaimedRun implements Claim { * its message, which can hold secrets or repository output, and rethrown so the task fails. */ async fail(error: unknown): Promise { - if (error instanceof Superseded) return await this.#record("cancelled", "superseded", "a newer run took over."); + if (error instanceof Superseded) return this.#record("cancelled", "superseded", "a newer run took over."); if (error instanceof ExpectedFailure) { await this.#publish("failed", error.message); - return await this.#record("failed", error.outcome, error.message); + return this.#record("failed", error.outcome, error.message); } if (error instanceof AgentError && AGENT_STOPS[error.code]) { // The agent's own message can quote repository output, so it goes to the event, not GitHub. const summary = `the ${error.agentId} agent ${AGENT_STOPS[error.code]}.`; await this.#publish("failed", summary); - return await this.#record("failed", "agent_stopped", summary, { error: error.message.slice(0, 500) }); + return this.#record("failed", "agent_stopped", summary, { error: error.message.slice(0, 500) }); } const summary = `something went wrong. Run \`${this.runId}\` has the details in the Render Dashboard.`; await this.#publish("failed", summary).catch(() => undefined); const message = error instanceof Error ? error.message : String(error); - await this.#record("failed", "error", summary, { error: message.slice(0, 500) }); + this.#record("failed", "error", summary, { error: message.slice(0, 500) }); throw error; } @@ -215,9 +210,9 @@ class ClaimedRun implements Claim { return await finishStatus(this.#status(state, summary, links)); } - async #record(state: TaskResult["state"], outcome: string, summary: string, extra: Record = {}): Promise { + #record(state: TaskResult["state"], outcome: string, summary: string, extra: Record = {}): TaskResult { const durationMs = Date.now() - this.#startedAt; - await emit({ ...this.#event, type: "run.finished", payload: { stage: this.#opts.task, state, outcome, durationMs, ...extra } }); + emit({ ...this.#event, type: "run.finished", payload: { stage: this.#opts.task, state, outcome, durationMs, ...extra } }); return { state, outcome, summary, traceId: this.traceId, runId: this.runId }; } } diff --git a/app/claims/index.ts b/app/claims/index.ts index 4f1b34a..996368b 100644 --- a/app/claims/index.ts +++ b/app/claims/index.ts @@ -7,6 +7,7 @@ export { type StatusGitHub, type StatusLink, type StatusUpdate, + type StopContext, type Subject, stopRun, subjectName, diff --git a/app/claims/status.ts b/app/claims/status.ts index 9afb779..7b9f804 100644 --- a/app/claims/status.ts +++ b/app/claims/status.ts @@ -1,4 +1,5 @@ import { ClientError, Render } from "@renderinc/sdk"; +import { emit } from "../events/index.js"; import { findMarkedComment, type GitHubClient, type IssueComment, type RunState, renderMarker, type StatusMarker } from "../github/index.js"; import { sandboxClient } from "../sandbox/index.js"; @@ -52,14 +53,32 @@ export async function readStatus(gh: StatusGitHub, number: number): Promise<{ co return findMarkedComment(await gh.listIssueComments(number), "status", await gh.botLogin()); } +/** Who stops a run and why, for the `run.finished` event recorded on the stopped run's behalf. */ +export interface StopContext { + /** "owner/name" */ + repo: string; + /** "issue#12" or "pr#34" */ + subject: string; + /** The task run doing the stopping. */ + by: string; + outcome: "superseded" | "cancelled"; +} + /** * Stops the run a status marker records: cancels it if it's still active, then terminates the sandbox it recorded. * Render kills a cancelled task without running its `finally`, so the sandbox would otherwise live until its - * timeout. Returns whether the run was still active. + * timeout, and the run never records how it ended; this records `run.finished` for it. Returns whether the run + * was still active. */ -export async function stopRun(runs: RunControl, status: StatusMarker): Promise { +export async function stopRun(runs: RunControl, status: StatusMarker, context: StopContext): Promise { const active = await isRunActive(runs, status.runId); - if (active) await cancelActiveRun(runs, status.runId); + if (active) { + await cancelActiveRun(runs, status.runId); + const { repo, subject, by, outcome } = context; + // Claiming tasks are started by the gateway, so each is its own root task run and its run ID is its trace ID. + const payload = { stage: status.stage, state: "cancelled", outcome, stoppedBy: by }; + emit({ traceId: status.runId, runId: status.runId, repo, subject, type: "run.finished", payload }); + } if (status.sandboxId) { const { sandboxId } = status; await runs.terminateSandbox(sandboxId).catch((error) => warn("could not terminate the stopped run's sandbox", error, { sandboxId })); @@ -94,16 +113,19 @@ async function cancelActiveRun(runs: RunControl, taskRunId: string): Promise { - const { gh, subject, task, title, runId } = opts; + const { gh, repo, subject, task, title, runId } = opts; const current = await readStatus(gh, subject.number); const previous = current && current.data.runId !== runId ? current.data : null; - const cancelledPrevious = previous ? await stopRun(opts.runs ?? workflowRuns(), previous) : false; + const stop = { repo, subject: subjectName(subject), by: runId, outcome: "superseded" } as const; + const cancelledPrevious = previous ? await stopRun(opts.runs ?? workflowRuns(), previous, stop) : false; const body = statusBody({ task, title, runId, state: "running" }); const comment = current ? await gh.updateComment(current.comment.id, body) : await gh.createComment(subject.number, body); await addWorkingLabel(gh, subject.number); diff --git a/app/entrypoints/gateway.ts b/app/entrypoints/gateway.ts index e168516..6c92ab9 100644 --- a/app/entrypoints/gateway.ts +++ b/app/entrypoints/gateway.ts @@ -1,7 +1,6 @@ import { serve } from "@hono/node-server"; import { Render } from "@renderinc/sdk"; import { requireEnv } from "../config/index.js"; -import { closeEvents } from "../events/index.js"; import { createGateway } from "../gateway/index.js"; const port = Number(process.env.PORT ?? 3000); @@ -23,8 +22,6 @@ console.log(`gateway listening on 0.0.0.0:${port}`); for (const signal of ["SIGINT", "SIGTERM"] as const) { process.once(signal, () => { - server.close(() => { - void closeEvents().finally(() => process.exit(0)); - }); + server.close(() => process.exit(0)); }); } diff --git a/app/events/index.ts b/app/events/index.ts index 4c87511..d7f25f9 100644 --- a/app/events/index.ts +++ b/app/events/index.ts @@ -1,66 +1,25 @@ -import pg from "pg"; - -/** One row in the append-only `events` table: telemetry only. Nothing reads it back to decide what a run does. */ +/** One telemetry event: how a task run or an agent run ended. Nothing reads these back to decide what a run does. */ export interface FactoryEvent { - /** The root task run of the task (or the gateway's delivery id for deliveries that started nothing). */ + /** The root task run of the task. */ traceId: string; - /** The task run that emitted the event, when there is one. */ + /** The task run the event describes. */ runId?: string; /** "owner/name" */ repo?: string; /** "issue#12" or "pr#34" */ subject?: string; - /** Dotted name: delivery.received, run.started, run.finished, agent.started, agent.finished, cost, ... */ - type: string; - payload?: Record; -} - -let pool: pg.Pool | undefined; - -function db(): pg.Pool | undefined { - if (pool) return pool; - const connectionString = process.env.DATABASE_URL?.trim(); - if (!connectionString) return undefined; - pool = new pg.Pool({ - connectionString, - max: 3, - application_name: "software-factory", - connectionTimeoutMillis: 3_000, - idleTimeoutMillis: 10_000, - allowExitOnIdle: true, - options: "-c statement_timeout=5000 -c lock_timeout=2000", - }); - // An idle client losing its connection must not crash the process. - pool.on("error", (error) => warn("events pool error", error)); - return pool; + type: "run.finished" | "agent.finished"; + payload: Record; } -function warn(msg: string, error: unknown, fields: Record = {}): void { - const message = error instanceof Error ? error.message : String(error); - console.error(JSON.stringify({ level: "warn", msg, ...fields, error: message })); -} - -/** Appends one event. Never throws and never blocks a run on the database; a no-op without DATABASE_URL. */ -export async function emit(event: FactoryEvent): Promise { +/** Writes the event to stdout as one JSON line, where Render logs and log streams pick it up. Never throws. */ +export function emit(event: FactoryEvent): void { + const { type, payload, ...ids } = event; try { - const p = db(); - if (!p) return; - await p.query("insert into events (trace_id, run_id, repo, subject, type, payload) values ($1, $2, $3, $4, $5, $6)", [ - event.traceId, - event.runId ?? null, - event.repo ?? null, - event.subject ?? null, - event.type, - JSON.stringify(event.payload ?? {}), - ]); + console.log(JSON.stringify({ level: "info", event: type, ...ids, ...payload })); } catch (error) { - warn("event dropped", error, { type: event.type, traceId: event.traceId }); + console.error( + JSON.stringify({ level: "warn", msg: "event dropped", event: type, error: error instanceof Error ? error.message : String(error) }), + ); } } - -/** Ends the pool so scripts can exit. */ -export async function closeEvents(): Promise { - const p = pool; - pool = undefined; - await p?.end(); -} diff --git a/app/gateway/index.ts b/app/gateway/index.ts index 073ba31..9adf3ed 100644 --- a/app/gateway/index.ts +++ b/app/gateway/index.ts @@ -1,18 +1,19 @@ import { Hono } from "hono"; import type { TaskName } from "../config/index.js"; -import { emit as defaultEmit } from "../events/index.js"; import { readBody } from "./body.js"; import { route } from "./routes.js"; import { verifySignature } from "./signature.js"; export interface GatewayDeps { startTask(task: TaskName, input: object, idempotencyKey: string): Promise<{ id: string }>; - emit?: typeof defaultEmit; webhookSecret: string; } +/** + * The webhook gateway. Every response says what happened to the delivery; GitHub keeps them under the App's + * Recent Deliveries, so only failures the gateway can't explain there are logged. + */ export function createGateway(deps: GatewayDeps): Hono { - const emit = deps.emit ?? defaultEmit; const app = new Hono(); app.get("/health", (c) => c.json({ status: "ok" })); @@ -21,68 +22,36 @@ export function createGateway(deps: GatewayDeps): Hono { app.post("/webhooks/github", async (c) => { const event = c.req.header("x-github-event") ?? ""; const deliveryId = c.req.header("x-github-delivery") ?? ""; - const delivered = (fields: { - outcome: string; - reason?: string; - action?: string; - repo?: string; - subject?: string; - runId?: string; - task?: string; - }) => { - const { repo, subject, runId, ...rest } = fields; - // Not awaited: a slow or absent database must not delay the response to GitHub. - void emit({ - traceId: runId ?? (deliveryId || crypto.randomUUID()), - runId, - repo, - subject, - type: "delivery.received", - payload: { event, deliveryId: deliveryId || null, ...rest }, - }); - }; - // Unauthenticated requests are answered without an event, so they can't write to the database. const body = await readBody(c.req.raw); if (body === null) return c.json({ error: "payload too large" }, 413); if (!verifySignature(body, c.req.header("x-hub-signature-256"), deps.webhookSecret)) { return c.json({ error: "invalid signature" }, 401); } - if (!deliveryId) { - delivered({ outcome: "rejected", reason: "missing delivery id" }); - return c.json({ error: "missing x-github-delivery" }, 400); - } + if (!deliveryId) return c.json({ error: "missing x-github-delivery" }, 400); let payload: unknown; try { payload = JSON.parse(body); } catch { - delivered({ outcome: "rejected", reason: "invalid json" }); return c.json({ error: "invalid json" }, 400); } - const action = stringAt(payload, "action"); - const repo = stringAt(payload, "repository", "full_name"); const decision = route(event, payload, deliveryId); - if (decision.kind === "ignore") { - delivered({ outcome: "ignored", reason: decision.reason, action, repo }); - return c.json({ ignored: decision.reason }); - } + if (decision.kind === "ignore") return c.json({ ignored: decision.reason }); if (decision.kind === "invalid") { const { task, subject, reason } = decision; - delivered({ outcome: "invalid", reason, action, repo, subject, task }); + warn("authenticated delivery produced an invalid task input", { deliveryId, event, task, subject, reason }); return c.json({ error: "invalid task input", task }, 422); } const { task, subject, idempotencyKey, input } = decision; try { const run = await deps.startTask(task, input, idempotencyKey); - delivered({ outcome: "started", action, repo, subject, task, runId: run.id }); return c.json({ runId: run.id, task }, 202); } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - delivered({ outcome: "start failed", reason, action, repo, subject, task }); + warn("could not start task", { deliveryId, event, task, subject, error: error instanceof Error ? error.message : String(error) }); // A 5xx marks the delivery failed in GitHub, where it can be redelivered by hand. return c.json({ error: "could not start task", task }, 502); } @@ -91,11 +60,6 @@ export function createGateway(deps: GatewayDeps): Hono { return app; } -function stringAt(value: unknown, ...path: string[]): string | undefined { - let current = value; - for (const key of path) { - if (typeof current !== "object" || current === null) return undefined; - current = (current as Record)[key]; - } - return typeof current === "string" ? current : undefined; +function warn(msg: string, fields: Record): void { + console.error(JSON.stringify({ level: "warn", msg, ...fields })); } diff --git a/app/runner/session.ts b/app/runner/session.ts index ffe9bb9..c1a0228 100644 --- a/app/runner/session.ts +++ b/app/runner/session.ts @@ -200,8 +200,16 @@ export async function runSession(opts: SessionOptions): Promise { onAccepted: (output) => { submission = { output }; }, - onRejected: (attempts) => { - if (attempts >= MAX_SUBMIT_ATTEMPTS) stop({ code: "invalid_submission", message: `${SUBMIT_TOOL} was rejected ${attempts} times` }); + onRejected: (attempts, problems, fields) => { + if (attempts < MAX_SUBMIT_ATTEMPTS) return; + // Schema paths, rule names, and field names only: TypeBox's errors don't quote the submitted values. + const last = problems.slice(0, 5).join("; "); + const names = + fields + .slice(0, 20) + .map((f) => f.slice(0, 40)) + .join(", ") || "none"; + stop({ code: "invalid_submission", message: `${SUBMIT_TOOL} was rejected ${attempts} times; last: ${last}; submitted fields: ${names}` }); }, }), ], diff --git a/app/runner/submit.ts b/app/runner/submit.ts index fb095fc..352000f 100644 --- a/app/runner/submit.ts +++ b/app/runner/submit.ts @@ -40,8 +40,8 @@ function checkSubmission(schema: JsonOutputSchema, candidate: unknown): Verdict interface SubmitHandlers { onAccepted(output: unknown): void; - /** Called with the number of rejected attempts so far. */ - onRejected(attempts: number): void; + /** Called with the number of rejected attempts so far, what was wrong with the latest one, and its top-level field names. */ + onRejected(attempts: number, problems: readonly string[], fields: readonly string[]): void; } /** The tool's parameters as the model sees them: the output schema without JSON Schema metadata. */ @@ -79,7 +79,8 @@ export function createSubmitTool(schema: JsonOutputSchema, handlers: SubmitHandl } rejected++; const remaining = MAX_SUBMIT_ATTEMPTS - rejected; - handlers.onRejected(rejected); + const fields = args && typeof args === "object" && !Array.isArray(args) ? Object.keys(args) : []; + handlers.onRejected(rejected, verdict.problems, fields); throw new Error( `${SUBMIT_TOOL} rejected (attempt ${rejected} of ${MAX_SUBMIT_ATTEMPTS}); nothing was recorded. ` + (remaining > 0 ? "Fix every problem below and call it again:\n" : "No attempts remain:\n") + diff --git a/app/schemas/implementation.ts b/app/schemas/implementation.ts index 47902cd..0d32d10 100644 --- a/app/schemas/implementation.ts +++ b/app/schemas/implementation.ts @@ -1,14 +1,14 @@ import { z } from "zod"; /** One verify command as the implementer ran it. Self-reported: the workflow reruns verify and trusts only its own run. */ -export const VerificationRun = z.object({ +export const VerificationRun = z.strictObject({ command: z.string().min(1), passed: z.boolean(), notes: z.string().max(2000).describe("What failed and what you did about it, or why you could not run it. Empty when it passed cleanly."), }); /** The implementer's submission, for a first run and for a repair run alike. */ -export const Implementation = z.object({ +export const Implementation = z.strictObject({ summary: z.string().min(1).max(4000).describe("What you changed and why, for the pull request description."), filesChanged: z.array(z.string().min(1)).max(500).describe("Repository-relative paths you created, modified, or deleted."), verification: z.array(VerificationRun).describe("Every verify command, in order, with its last result."), diff --git a/app/schemas/review.ts b/app/schemas/review.ts index 26f0993..f318567 100644 --- a/app/schemas/review.ts +++ b/app/schemas/review.ts @@ -3,7 +3,7 @@ import { z } from "zod"; export const SEVERITIES = ["critical", "high", "medium", "low"] as const; export const Severity = z.enum(SEVERITIES); -export const ReviewComment = z.object({ +export const ReviewComment = z.strictObject({ file: z.string().min(1).describe("Repository-relative path of a file the pull request changes."), line: z.number().int().min(1).describe("Line number in the new version of the file."), severity: Severity, @@ -12,7 +12,7 @@ export const ReviewComment = z.object({ }); /** The PR reviewer's submission. It summarizes and assesses risk; it never approves or requests changes. */ -export const Review = z.object({ +export const Review = z.strictObject({ summary: z.string().min(1).max(8000).describe("Markdown overview of the review, including findings that don't fit an inline comment."), riskTier: z.enum(["trivial", "low", "medium", "high"]).describe("The overall assessment."), comments: z.array(ReviewComment).max(50).describe("Actionable inline comments, most important first."), diff --git a/app/schemas/spec.ts b/app/schemas/spec.ts index 2135ad0..ed7d8d1 100644 --- a/app/schemas/spec.ts +++ b/app/schemas/spec.ts @@ -3,19 +3,19 @@ import { z } from "zod"; const Item = z.string().min(1).max(1000); const Items = (max: number) => z.array(Item).max(max); -export const AcceptanceCriterion = z.object({ +export const AcceptanceCriterion = z.strictObject({ id: z.string().regex(/^AC-[1-9]\d*$/, 'criterion IDs look like "AC-1"'), text: z.string().min(1).max(1000), /** No browser testing at launch: a criterion is checked by tests or the build, or by a person. */ verification: z.enum(["automated", "manual"]), }); -export const AffectedArea = z.object({ +export const AffectedArea = z.strictObject({ path: z.string().min(1).max(300).describe("Repository-relative path. New files are fine; say so in change."), change: z.string().min(1).max(1000), }); -export const Spec = z.object({ +export const Spec = z.strictObject({ title: z.string().min(1).max(200), summary: z.string().min(1).max(2000), problem: z.string().max(4000), @@ -31,13 +31,13 @@ export const Spec = z.object({ }); /** What the spec writer learned, kept with the spec so later rounds skip the research. */ -export const SpecNotes = z.object({ +export const SpecNotes = z.strictObject({ findings: z.string().max(8000), relevantPaths: z.array(z.string().min(1)).max(50), }); /** The spec writer's submission. */ -export const SpecDraft = z.object({ +export const SpecDraft = z.strictObject({ spec: Spec, notes: SpecNotes, changeSummary: z.string().max(2000).optional().describe("Only when revising a previous version: what changed and why."), diff --git a/app/schemas/triage.ts b/app/schemas/triage.ts index 356c232..706be6f 100644 --- a/app/schemas/triage.ts +++ b/app/schemas/triage.ts @@ -2,7 +2,7 @@ import { z } from "zod"; /** The triage agent's submission: whether a competent engineer would know what to build. */ export const TriageDecision = z - .object({ + .strictObject({ decision: z .enum(["ask", "draft", "decline"]) .describe('"draft" by default; "ask" only when the intent is unclear; "decline" for spam, duplicates, or out of scope.'), diff --git a/app/sql/0001_events.sql b/app/sql/0001_events.sql deleted file mode 100644 index 39b3a3a..0000000 --- a/app/sql/0001_events.sql +++ /dev/null @@ -1,13 +0,0 @@ -create table events ( - id bigserial primary key, - trace_id text not null, - run_id text, - repo text, - subject text, - type text not null, - payload jsonb not null default '{}', - created_at timestamptz not null default now() -); -create index events_current_run_claim - on events (repo, subject, id desc) - where type = 'run.claimed'; diff --git a/app/workflows/cancel.ts b/app/workflows/cancel.ts index b9c0104..b5cd8e1 100644 --- a/app/workflows/cancel.ts +++ b/app/workflows/cancel.ts @@ -25,7 +25,6 @@ export async function cancelRun( const gh = deps.gh ?? githubFor(repo); const { traceId, runId } = traceIds(ctx); const event = { traceId, runId, repo: repo.fullName, subject: `${input.kind}#${input.number}` }; - await emit({ ...event, type: "run.started", payload: { stage: TASKS.cancel, actor: input.actor } }); const result = await (async (): Promise => { if (!(await checkPermission(gh, input.actor))) { @@ -36,7 +35,8 @@ export async function cancelRun( return { cancelled: false, reason: "permission_denied" }; } const current = await readStatus(gh, input.number); - if (!current || !(await stopRun(deps.runs ?? workflowRuns(), current.data))) { + const stop = { repo: repo.fullName, subject: event.subject, by: runId, outcome: "cancelled" } as const; + if (!current || !(await stopRun(deps.runs ?? workflowRuns(), current.data, stop))) { await gh.createComment(input.number, `@${input.actor} no factory run is in progress here.`); return { cancelled: false, reason: "nothing_running" }; } @@ -46,7 +46,7 @@ export async function cancelRun( return { cancelled: true, runId: cancelled }; })(); - await emit({ ...event, type: "run.finished", payload: { stage: TASKS.cancel, state: "done", ...result } }); + emit({ ...event, type: "run.finished", payload: { stage: TASKS.cancel, state: "done", ...result } }); return result; } diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index af1565f..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: software_factory - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - "55432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d software_factory"] - interval: 5s - timeout: 3s - retries: 10 diff --git a/docs/architecture.md b/docs/architecture.md index 65064b6..1a0d3fc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,8 +10,6 @@ decides what may be written to GitHub. flowchart LR GitHub -->|signed webhook| Gateway Gateway -->|start task| Workflows - Gateway -.->|delivery telemetry| Postgres - Workflows -.->|run telemetry| Postgres Workflows -->|restore and execute| Sandbox Sandbox -->|proposed result| Workflows Workflows -->|validated writes| GitHub @@ -19,13 +17,11 @@ flowchart LR Snapshot -.->|restore| Sandbox ``` -The Blueprint creates three persistent resources: +The Blueprint creates two services: - **Gateway** verifies GitHub webhooks and maps each delivery to one task. - **Workflows** runs the claiming tasks, agents, cancellation, snapshot refresh, and GitHub publishing. -- **Postgres** stores best-effort telemetry. It's optional: without - `DATABASE_URL`, events are dropped and nothing else changes. Each claiming task receives a fresh Sandbox restored from a repository snapshot. Render Workflows owns task state and timeouts. GitHub owns the human @@ -71,9 +67,9 @@ tests. | Current run for an issue or PR, and its Sandbox | App-authored status comment marker | | User-visible run status | The same status comment, and the working label | | Prepared repository filesystem | Render Sandbox snapshot | -| Delivery, run, agent, and cost history | Postgres events (optional) | +| Run and agent outcomes, usage, and cost | JSON lines in the service logs | -Nothing reads Postgres back to decide what a task does. +Telemetry is write-only: nothing reads it back to decide what a task does. ## Idempotency and competing runs @@ -176,7 +172,6 @@ Changes must preserve these properties: 7. End automation at a draft PR and COMMENT review. 8. Implement the exact specification that was approved. 9. Put a timeout and `finally` cleanup around every Sandbox. -10. Keep Postgres telemetry-only: no task behavior may depend on it. Tests pin these invariants and each built-in agent's tool authority. diff --git a/docs/configuration.md b/docs/configuration.md index 0ac4585..2046b12 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,14 +108,13 @@ budget. ## Environment variables -Every local variable appears in [`.env.example`](../.env.example). Deployed -database and workflow references are wired by the Blueprint. +Every local variable appears in [`.env.example`](../.env.example). The +Blueprint wires the gateway's reference to the Workflows service. ### Gateway | Variable | Purpose | |---|---| -| `DATABASE_URL` | Optional: migrations and delivery telemetry | | `GITHUB_WEBHOOK_SECRET` | Verify GitHub webhook signatures | | `RENDER_API_KEY` | Start Workflows tasks | | `RENDER_WORKFLOW_SLUG` | Address the deployed Workflows service | @@ -125,7 +124,6 @@ database and workflow references are wired by the Blueprint. | Variable | Purpose | |---|---| -| `DATABASE_URL` | Optional: run and agent telemetry | | `RENDER_API_KEY` | Manage Sandboxes, snapshots, and task cancellation | | `RENDER_WORKSPACE_ID` | Workspace that owns Sandboxes and snapshots | | `GITHUB_APP_ID` | Authenticate the GitHub App | diff --git a/docs/design-notes.md b/docs/design-notes.md index 918c6d8..10717a1 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -7,8 +7,8 @@ not required for deployment or operation. Render Workflows owns task execution. GitHub owns collaboration artifacts, including the status comment that says which run currently works on an issue -or pull request. Postgres only records telemetry, so the factory runs without -it. Every additional source of truth creates reconciliation work. +or pull request. There is no database; telemetry goes to logs. Every +additional source of truth creates reconciliation work. ## Make approval an explicit transition diff --git a/docs/operations.md b/docs/operations.md index efd3827..3ffbd33 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,18 +10,19 @@ factory's status comment shows progress, the current run ID, and final links. Find that run under `factory-workflows` in the Render Dashboard to inspect logs, inputs, and its result. -When `DATABASE_URL` is set, events are also appended to Postgres. To inspect -one run: - -```sql -select created_at, type, subject, payload -from events -where trace_id = '' -order by id; +Every task run ends with a `run.finished` JSON line, and every agent run with +an `agent.finished` line, in the `factory-workflows` logs. Search for them, or +send them to a log stream: + +```bash +render logs -r --text agent.finished ``` -Telemetry includes task outcomes and agent model, turns, token usage, and -reported cost. Task state remains in Render Workflows; GitHub remains the +`run.finished` carries the task, state, outcome, and duration. `agent.finished` +carries the agent, model, turns, token usage, reported cost, and, when an agent +is stopped, why. A run stopped by a newer run or by `/factory cancel` gets its +`run.finished` line from whatever stopped it, because Render kills a cancelled +run before it can write one. Task state remains in Render Workflows; GitHub remains the source of truth for issues, specifications, pull requests, and reviews. ## Cancel or restart @@ -133,7 +134,7 @@ mismatch. - PR Reviewer executes the labeled PR's configured verification commands. - Sandboxes use allow-all egress. - Local filesystem writes on Render services are ephemeral; durable state - belongs in GitHub, Workflows, Postgres, or Sandbox snapshots. + belongs in GitHub, Workflows, or Sandbox snapshots. Review [Architecture](architecture.md#residual-risks) before using private code or untrusted pull requests. diff --git a/docs/quickstart.md b/docs/quickstart.md index ecd4718..16dc83f 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -14,9 +14,8 @@ only you should take. - admin access to each target GitHub repository; and - the Node.js version in [`.node-version`](../.node-version). -The Blueprint creates paid Render resources: a Starter web service, a -Workflows service, and a Basic-256mb Postgres database. Sandbox and model -usage are charged separately. +The Blueprint creates paid Render resources: a Starter web service and a +Workflows service. Sandbox and model usage are charged separately. ## What the setup agent will do diff --git a/docs/runbooks/setup.md b/docs/runbooks/setup.md index 0a87883..27634f7 100644 --- a/docs/runbooks/setup.md +++ b/docs/runbooks/setup.md @@ -13,7 +13,7 @@ repository, and complete one Triage run. Setup is complete only when: -- `factory-gateway`, `factory-workflows`, and `factory-db` are live; +- `factory-gateway` and `factory-workflows` are live; - the GitHub App is installed on every configured repository; - the App webhook is active and points to the gateway; - the factory labels exist; @@ -239,9 +239,8 @@ The user confirms installation on every exact configured repository. Explain that [`render.yaml`](../../render.yaml) creates paid resources in Oregon: -- `factory-gateway`, a Starter web service; -- `factory-workflows`, a Workflows service; and -- `factory-db`, a Basic-256mb Postgres database. +- `factory-gateway`, a Starter web service; and +- `factory-workflows`, a Workflows service. The user should create the Blueprint from the configured fork and deployment branch. @@ -269,7 +268,7 @@ into chat. The Render workspace ID starts with `tea-`. ### Pass condition -All three resources are created, the gateway reports live, and the Workflows +Both services are created, the gateway reports live, and the Workflows service has deployed successfully. ### Recovery @@ -277,8 +276,6 @@ service has deployed successfully. - Blank `sync: false` field: set the missing value and redeploy. - Build failure: inspect the failing service's build log before changing configuration. -- Pre-deploy migration failure: verify the Blueprint's `DATABASE_URL` wiring - and database health. - Port or health failure: confirm the gateway binds to `0.0.0.0:$PORT` and serves `/ready`. @@ -362,10 +359,9 @@ Both commands exit successfully with every required check passing. ### Recovery -- Gateway failure: check webhook secret, Render API key, workflow slug, and - database connection. +- Gateway failure: check webhook secret, Render API key, and workflow slug. - Workflows failure: check Render workspace ID, Render API key, GitHub App - credentials, provider key, database, and registered tasks. + credentials, provider key, and registered tasks. - App failure: compare required permissions and events with Phase 4. - Label or installation failure: rerun `npm run setup:github` after fixing App access. diff --git a/package-lock.json b/package-lock.json index 4204768..0466971 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,6 @@ "@renderinc/sdk": "^1.3.0", "esbuild": "^0.28.2", "hono": "^4.0.0", - "pg": "^8.23.0", "tar": "^7.5.22", "tsx": "^4.19.0", "typebox": "1.3.7", @@ -24,7 +23,6 @@ "devDependencies": { "@biomejs/biome": "^2.5.11", "@types/node": "^22.0.0", - "@types/pg": "^8.23.1", "typescript": "^5.5.0", "vitest": "^4.1.11" }, @@ -1805,18 +1803,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/pg": { - "version": "8.23.1", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", - "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, "node_modules/@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", @@ -2812,95 +2798,6 @@ "dev": true, "license": "MIT" }, - "node_modules/pg": { - "version": "8.23.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", - "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", - "license": "MIT", - "dependencies": { - "pg-connection-string": "^2.14.0", - "pg-pool": "^3.14.0", - "pg-protocol": "^1.16.0", - "pg-types": "2.2.0", - "pgpass": "1.0.5" - }, - "engines": { - "node": ">= 16.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.4.0" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", - "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", - "license": "MIT", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", - "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", - "license": "MIT" - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-pool": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", - "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", - "license": "MIT", - "peerDependencies": { - "pg": ">=8.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.0.tgz", - "integrity": "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "license": "MIT", - "dependencies": { - "split2": "^4.1.0" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -2950,45 +2847,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", - "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/protobufjs": { "version": "7.6.6", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", @@ -3104,15 +2962,6 @@ "node": ">=0.10.0" } }, - "node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, "node_modules/stackback": { "version": "0.0.2", "dev": true, @@ -3446,15 +3295,6 @@ } } }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, "node_modules/yallist": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", diff --git a/package.json b/package.json index 0a62257..4755ba1 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,6 @@ "start:workflows": "tsx app/entrypoints/workflows.ts", "dev:gateway": "node --env-file=.env node_modules/.bin/tsx watch app/entrypoints/gateway.ts", "dev:workflows": "render workflows dev -- node --env-file=.env node_modules/.bin/tsx app/entrypoints/workflows.ts", - "dev:postgres": "docker compose up postgres", - "db:migrate": "node --env-file-if-exists=.env --import tsx scripts/migrate.ts", "setup:app": "node --env-file-if-exists=.env --import tsx scripts/setup-app.ts", "setup:github": "node --env-file-if-exists=.env --import tsx scripts/setup-github.ts", "doctor": "node --env-file-if-exists=.env --import tsx scripts/doctor.ts", @@ -33,7 +31,6 @@ "@renderinc/sdk": "^1.3.0", "esbuild": "^0.28.2", "hono": "^4.0.0", - "pg": "^8.23.0", "tar": "^7.5.22", "tsx": "^4.19.0", "typebox": "1.3.7", @@ -42,7 +39,6 @@ "devDependencies": { "@biomejs/biome": "^2.5.11", "@types/node": "^22.0.0", - "@types/pg": "^8.23.1", "typescript": "^5.5.0", "vitest": "^4.1.11" } diff --git a/render.yaml b/render.yaml index 6a33411..5e2cd09 100644 --- a/render.yaml +++ b/render.yaml @@ -1,13 +1,8 @@ -# Software Factory: three resources. -# - factory-db: optional Postgres holding the append-only events table, used only for telemetry. -# - factory-gateway: verifies GitHub webhooks and starts the matching Workflows task. Migrations run before each deploy. -# - factory-workflows: runs every task (stages, agents, snapshot refresh, cancel). It has no `repo` field, so it +# Software Factory: two services. +# - factory-gateway: verifies GitHub webhooks and starts the matching Workflows task. +# - factory-workflows: runs every task (agents, snapshot refresh, cancel). It has no `repo` field, so it # deploys from the same repository as this Blueprint, including forks. - -databases: - - name: factory-db - region: oregon - plan: basic-256mb +# Run and agent telemetry is logged as JSON lines; find it in each service's logs or a log stream. services: - type: web @@ -17,13 +12,8 @@ services: plan: starter buildCommand: npm ci startCommand: npm run start:gateway - preDeployCommand: npm run db:migrate healthCheckPath: /ready envVars: - - key: DATABASE_URL - fromDatabase: - name: factory-db - property: connectionString - key: GITHUB_WEBHOOK_SECRET sync: false - key: RENDER_API_KEY @@ -41,10 +31,6 @@ services: buildCommand: npm ci startCommand: npm run start:workflows envVars: - - key: DATABASE_URL - fromDatabase: - name: factory-db - property: connectionString - key: RENDER_API_KEY sync: false - key: RENDER_WORKSPACE_ID diff --git a/scripts/doctor.ts b/scripts/doctor.ts index 4bec9da..ac51ef8 100644 --- a/scripts/doctor.ts +++ b/scripts/doctor.ts @@ -5,9 +5,6 @@ * * Checks the environment it runs in: locally that's .env; in a service's Render Shell it's that service's env. */ -import { readdir } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { config } from "../app/config/index.js"; import { runDoctor } from "./lib/doctor.js"; @@ -27,12 +24,9 @@ if (!services) { process.exit(2); } -const sqlDir = join(dirname(fileURLToPath(import.meta.url)), "..", "app", "sql"); -const migrations = (await readdir(sqlDir)).filter((name) => name.endsWith(".sql")).sort(); - const out = new Reporter(); try { - process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos), models: config.models, migrations }, out); + process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos), models: config.models }, out); } catch (error) { out.fail(`doctor crashed: ${errorMessage(error)}`); process.exitCode = 1; diff --git a/scripts/lib/doctor.ts b/scripts/lib/doctor.ts index ffed633..c449387 100644 --- a/scripts/lib/doctor.ts +++ b/scripts/lib/doctor.ts @@ -1,5 +1,4 @@ -/** doctor: read-only checks of one or both services' environment, the GitHub App, Postgres, and Workflows tasks. */ -import pg from "pg"; +/** doctor: read-only checks of one or both services' environment, the GitHub App, and Workflows tasks. */ import { TASKS } from "../../app/config/index.js"; import { PROVIDER_KEY_ENV } from "../../app/runner/keys.js"; import { appCredentialsFrom, type Env, GitHubApp, type ScriptDeps, splitRepo } from "./github.js"; @@ -8,31 +7,21 @@ import { errorMessage, type Reporter } from "./output.js"; import { LABELS, missingEvents, missingPermissions, SECRET_ENV, SERVICE_ENV, type Service, WEBHOOK_PATH } from "./requirements.js"; import { appSettingsUrl } from "./setup-github.js"; -export interface DbClient { - query(sql: string): Promise<{ rows: T[] }>; - end(): Promise; -} - export interface DoctorOptions extends ScriptDeps { services: readonly Service[]; repos: readonly string[]; /** config.models: "provider/model" per tier. */ models: Readonly>; - /** Migration filenames in app/sql. */ - migrations: readonly string[]; - connectDb?: (connectionString: string) => Promise; renderApiUrl?: string; } export async function runDoctor(opts: DoctorOptions, out: Reporter): Promise { const { env } = opts; out.addSecrets(...SECRET_ENV.map((name) => env[name]), ...modelKeys(opts.models).map((name) => env[name])); - out.addSecrets(databasePassword(env.DATABASE_URL)); checkEnv(opts, out); if (opts.services.includes("workflows")) checkModels(opts, out); await checkGitHub(opts, out); - await checkDatabase(opts, out); await checkTasks(opts, out); return out.summary(); } @@ -204,54 +193,6 @@ async function checkRepo(app: GitHubApp, info: { slug: string; html_url: string } } -async function checkDatabase(opts: DoctorOptions, out: Reporter): Promise { - out.heading("Postgres (optional telemetry)"); - const url = opts.env.DATABASE_URL?.trim(); - if (!url) { - out.skip("database checks: DATABASE_URL is not set, so telemetry is off"); - return; - } - let client: DbClient; - try { - client = await (opts.connectDb ?? connectPg)(url); - } catch (error) { - out.fail(`DATABASE_URL is unreachable: ${errorMessage(error)}`, "Use the internal URL on Render, the external one locally"); - return; - } - try { - out.pass("DATABASE_URL is reachable"); - const { rows } = await client.query<{ events: string | null; migrations: string | null }>( - "select to_regclass('public.events')::text as events, to_regclass('public.schema_migrations')::text as migrations", - ); - const tables = rows[0]; - if (tables?.events) out.pass("events table exists"); - else out.fail("events table is missing", "Run `npm run db:migrate` (the gateway's preDeployCommand)"); - if (!tables?.migrations) { - out.fail("schema_migrations table is missing", "Run `npm run db:migrate`"); - return; - } - const applied = new Set((await client.query<{ filename: string }>("select filename from schema_migrations")).rows.map((r) => r.filename)); - const pending = opts.migrations.filter((file) => !applied.has(file)); - if (pending.length) out.fail(`migrations not applied: ${pending.join(", ")}`, "Run `npm run db:migrate`"); - else out.pass(`all ${opts.migrations.length} migration(s) applied`); - } catch (error) { - out.fail(`database query failed: ${errorMessage(error)}`); - } finally { - await client.end().catch(() => {}); - } -} - -async function connectPg(connectionString: string): Promise { - const client = new pg.Client({ - connectionString, - application_name: "software-factory-doctor", - connectionTimeoutMillis: 5_000, - statement_timeout: 5_000, - }); - await client.connect(); - return client; -} - /** * The Render API's GET /v1/tasks, filtered by slug, for each task the gateway starts. The SDK's workflows client * has no list-tasks method, so this is a plain request. A non-auth API error skips rather than fails. @@ -305,8 +246,3 @@ function safeUrl(value: string): URL | null { return null; } } - -function databasePassword(url: string | undefined): string | undefined { - const parsed = url ? safeUrl(url.trim()) : null; - return parsed?.password ? decodeURIComponent(parsed.password) : undefined; -} diff --git a/scripts/lib/requirements.ts b/scripts/lib/requirements.ts index 96b4a73..ea2607d 100644 --- a/scripts/lib/requirements.ts +++ b/scripts/lib/requirements.ts @@ -54,14 +54,14 @@ export const LABELS: readonly LabelSpec[] = [ export type Service = "gateway" | "workflows"; -/** The env vars each service requires, as render.yaml declares them. DATABASE_URL is optional: it only enables telemetry. */ +/** Each service's env vars, exactly as render.yaml declares them. */ export const SERVICE_ENV: Readonly> = { gateway: ["GITHUB_WEBHOOK_SECRET", "RENDER_API_KEY", "RENDER_WORKFLOW_SLUG"], workflows: ["RENDER_API_KEY", "RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", "ANTHROPIC_API_KEY"], }; /** Values doctor must never print. */ -export const SECRET_ENV: readonly string[] = ["DATABASE_URL", "GITHUB_WEBHOOK_SECRET", "RENDER_API_KEY", "GITHUB_APP_PRIVATE_KEY"]; +export const SECRET_ENV: readonly string[] = ["GITHUB_WEBHOOK_SECRET", "RENDER_API_KEY", "GITHUB_APP_PRIVATE_KEY"]; const LEVELS = ["none", "read", "write", "admin"]; diff --git a/scripts/migrate.ts b/scripts/migrate.ts deleted file mode 100644 index 5d08130..0000000 --- a/scripts/migrate.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { readdir, readFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import pg from "pg"; - -const LOCK_ID = 0x736f66747761; // "softwa": stable advisory lock key for migrations - -const sqlDir = join(dirname(fileURLToPath(import.meta.url)), "..", "app", "sql"); -const files = (await readdir(sqlDir)).filter((name) => name.endsWith(".sql")).sort(); - -const connectionString = process.env.DATABASE_URL?.trim(); -if (!connectionString) { - console.log("DATABASE_URL is not set: telemetry is off and there is nothing to migrate"); - process.exit(0); -} - -// A dedicated client with no statement timeout: migrations may legitimately run long. -const client = new pg.Client({ - connectionString, - application_name: "software-factory-migrate", - connectionTimeoutMillis: 10_000, -}); -await client.connect(); - -try { - await client.query("select pg_advisory_lock($1)", [LOCK_ID]); - try { - await client.query(` - create table if not exists schema_migrations ( - filename text primary key, - applied_at timestamptz not null default now() - ) - `); - - const { rows } = await client.query<{ filename: string }>("select filename from schema_migrations"); - const applied = new Set(rows.map((row) => row.filename)); - - for (const file of files) { - if (applied.has(file)) continue; - console.log(`applying ${file}`); - const sql = await readFile(join(sqlDir, file), "utf8"); - await client.query("begin"); - try { - await client.query(sql); - await client.query("insert into schema_migrations (filename) values ($1)", [file]); - await client.query("commit"); - } catch (error) { - await client.query("rollback"); - throw error; - } - } - - console.log(`${files.length} migration file(s) checked, up to date`); - } finally { - await client.query("select pg_advisory_unlock($1)", [LOCK_ID]); - } -} finally { - await client.end(); -} diff --git a/tests/agents.test.ts b/tests/agents.test.ts index 962c594..08bbf82 100644 --- a/tests/agents.test.ts +++ b/tests/agents.test.ts @@ -85,8 +85,8 @@ describe("runAgent in a sandbox", () => { for (const { command } of sandbox.execs) expect(command).not.toContain(HOST_KEY); expect(sandbox.uploads.get(`${runDir}/input.json`)).not.toContain(HOST_KEY); - expect(emitted()[0]).toMatchObject({ type: "agent.started", payload: { agentId: "reviewer" } }); - expect(emitted()[1]).toMatchObject({ + expect(emitted()).toHaveLength(1); + expect(emitted()[0]).toMatchObject({ type: "agent.finished", traceId: "trace-2", payload: { outcome: "ok", turns: 2, model: "anthropic/claude-opus-5", costUsd: run.costUsd, usage: run.usage }, diff --git a/tests/cancel.test.ts b/tests/cancel.test.ts index 29f2617..9286c5b 100644 --- a/tests/cancel.test.ts +++ b/tests/cancel.test.ts @@ -3,7 +3,7 @@ import type { FactoryEvent } from "../app/events/index.js"; const events: FactoryEvent[] = []; vi.mock("../app/events/index.js", () => ({ - emit: async (event: FactoryEvent) => { + emit: (event: FactoryEvent) => { events.push(event); }, })); @@ -83,8 +83,12 @@ describe("cancel task", () => { expect(comments[0]?.body).toContain("**Spec-to-PR** was cancelled: cancelled by @alice."); expect(parseMarker(comments[0]?.body ?? "", "status")).toMatchObject({ runId: "trn-1", state: "cancelled" }); expect(labels.has(WORKING_LABEL)).toBe(false); - expect(events.map((e) => e.type)).toEqual(["run.started", "run.finished"]); - expect(events[0]).toMatchObject({ traceId: "trn-cancel", runId: "trn-cancel", repo: REPO }); + expect(events.map((e) => [e.type, e.runId])).toEqual([ + ["run.finished", "trn-1"], + ["run.finished", "trn-cancel"], + ]); + expect(events[0]?.payload).toMatchObject({ state: "cancelled", outcome: "cancelled", stoppedBy: "trn-cancel" }); + expect(events[1]).toMatchObject({ traceId: "trn-cancel", runId: "trn-cancel", repo: REPO }); }); it("refuses an actor without permission and leaves the run alone", async () => { diff --git a/tests/claims.test.ts b/tests/claims.test.ts index 24c8cb2..86296b7 100644 --- a/tests/claims.test.ts +++ b/tests/claims.test.ts @@ -3,7 +3,7 @@ import type { FactoryEvent } from "../app/events/index.js"; const events: FactoryEvent[] = []; vi.mock("../app/events/index.js", () => ({ - emit: async (event: FactoryEvent) => { + emit: (event: FactoryEvent) => { events.push(event); }, })); @@ -96,9 +96,18 @@ describe("takeOver", () => { const gh = new FakeGitHub(); const existing = gh.addStatus("trn-old", "running", "sbx-old"); const runs = new FakeRuns({ "trn-old": "running" }); - const result = await takeOver({ gh, subject, task: "triage", title: "Triage", runId: "trn-new", runs }); + const result = await takeOver({ gh, repo: REPO.fullName, subject, task: "triage", title: "Triage", runId: "trn-new", runs }); expect(runs.cancelled).toEqual(["trn-old"]); expect(runs.terminated).toEqual(["sbx-old"]); + // The stopped run is killed before it can record how it ended, so the takeover records it. + expect(events).toContainEqual({ + traceId: "trn-old", + runId: "trn-old", + repo: REPO.fullName, + subject: "issue#12", + type: "run.finished", + payload: { stage: "triage", state: "cancelled", outcome: "superseded", stoppedBy: "trn-new" }, + }); expect(result).toEqual({ commentId: existing.id, previousRunId: "trn-old", cancelledPrevious: true }); expect(gh.comments).toHaveLength(1); expect(parseMarker(gh.comments[0]?.body ?? "", "status")).toMatchObject({ runId: "trn-new", state: "running", stage: "triage" }); @@ -131,8 +140,7 @@ describe("withClaim", () => { expect(gh.comments[0]?.body).toContain("did not run: @reader needs `write` permission"); expect(parseMarker(gh.comments[0]?.body ?? "", "status")?.state).toBe("rejected"); expect(gh.labels.has(WORKING_LABEL)).toBe(false); - expect(events.map((e) => e.type)).toEqual(["run.started", "run.finished"]); - expect(events.at(-1)?.payload).toMatchObject({ state: "rejected" }); + expect(events).toMatchObject([{ type: "run.finished", traceId: "trn-root", runId: "trn-run", payload: { state: "rejected" } }]); }); it("runs the work and records events with the trace id and outcome", async () => { @@ -159,15 +167,15 @@ describe("withClaim", () => { expect(gh.comments[0]?.body).toContain("**Triage** finished: asked 2 questions"); expect(gh.labels.has(WORKING_LABEL)).toBe(false); expect(events).toMatchObject([ + { type: "run.finished", traceId: "trn-prev", payload: { state: "cancelled", outcome: "superseded", stoppedBy: "trn-run" } }, { - type: "run.started", + type: "run.finished", traceId: "trn-root", runId: "trn-run", repo: REPO.fullName, subject: "issue#12", - payload: { stage: "triage", actor: "maintainer" }, + payload: { stage: "triage", state: "done", outcome: "asked-questions" }, }, - { type: "run.finished", traceId: "trn-root", payload: { state: "done", outcome: "asked-questions" } }, ]); }); diff --git a/tests/events.test.ts b/tests/events.test.ts index 2fd270c..987e44a 100644 --- a/tests/events.test.ts +++ b/tests/events.test.ts @@ -1,52 +1,39 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { emit } from "../app/events/index.js"; -const pg = vi.hoisted(() => ({ - query: vi.fn(), - end: vi.fn(async () => {}), - constructed: 0, -})); - -vi.mock("pg", () => { - class Pool { - constructor() { - pg.constructed++; - } - query = pg.query; - end = pg.end; - on() { - return this; - } - } - return { default: { Pool } }; -}); - -const { closeEvents, emit } = await import("../app/events/index.js"); - -const event = { traceId: "trn-1", type: "run.started", repo: "acme/web", payload: { a: 1 } }; - -beforeEach(() => { - pg.query.mockReset(); - pg.constructed = 0; -}); - -afterEach(async () => { - await closeEvents(); - vi.unstubAllEnvs(); +afterEach(() => { vi.restoreAllMocks(); }); describe("emit", () => { - it("swallows database errors and logs one JSON warning", async () => { - vi.stubEnv("DATABASE_URL", "postgres://fake/db"); - pg.query.mockRejectedValueOnce(new Error("connection refused")); - const stderr = vi.spyOn(console, "error").mockImplementation(() => {}); - await expect(emit(event)).resolves.toBeUndefined(); - expect(stderr).toHaveBeenCalledTimes(1); - expect(JSON.parse(stderr.mock.calls[0]?.[0] as string)).toMatchObject({ - level: "warn", - msg: "event dropped", - type: "run.started", - error: "connection refused", + it("writes one JSON line to stdout with the event name, its ids, and its payload", () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + emit({ + traceId: "trn-root", + runId: "trn-2", + repo: "acme/web", + subject: "issue#12", + type: "run.finished", + payload: { stage: "triage", state: "done", outcome: "drafted" }, + }); + expect(log).toHaveBeenCalledTimes(1); + expect(JSON.parse(log.mock.calls[0]?.[0] as string)).toEqual({ + level: "info", + event: "run.finished", + traceId: "trn-root", + runId: "trn-2", + repo: "acme/web", + subject: "issue#12", + stage: "triage", + state: "done", + outcome: "drafted", }); }); + + it("never throws, even when the payload can't be serialized", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + const warn = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(() => emit({ traceId: "trn-1", type: "agent.finished", payload: { cost: 1n } })).not.toThrow(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("event dropped")); + }); }); diff --git a/tests/gateway.test.ts b/tests/gateway.test.ts index 12e46c0..2ab2390 100644 --- a/tests/gateway.test.ts +++ b/tests/gateway.test.ts @@ -2,7 +2,6 @@ import { createHmac } from "node:crypto"; import { beforeEach, describe, expect, it, type Mock, vi } from "vitest"; import { config, type TaskName } from "../app/config/index.js"; import { TriageInput } from "../app/schemas/tasks.js"; -import type { FactoryEvent } from "../app/events/index.js"; import { createGateway, type GatewayDeps } from "../app/gateway/index.js"; const SECRET = "webhook-secret"; @@ -30,38 +29,28 @@ function webhook(body: string, { event = "issues", delivery = "delivery-1" as st } let startTask: Mock; -let emit: Mock<(event: FactoryEvent) => Promise>; let app: ReturnType; beforeEach(() => { startTask = vi.fn(async (_task: TaskName, _input: object, _key: string) => ({ id: "trn-123" })); - emit = vi.fn(async (_event: FactoryEvent) => {}); - app = createGateway({ startTask, emit, webhookSecret: SECRET }); + app = createGateway({ startTask, webhookSecret: SECRET }); }); -const deliveryEvent = () => { - expect(emit).toHaveBeenCalledTimes(1); - const event = emit.mock.calls[0]?.[0] as FactoryEvent; - expect(event.type).toBe("delivery.received"); - return event; -}; - describe("gateway", () => { - it("serves /health and /ready without the database", async () => { + it("serves /health and /ready", async () => { expect((await app.request("/health")).status).toBe(200); const ready = await app.request("/ready"); expect(ready.status).toBe(200); expect(await ready.json()).toEqual({ status: "ready" }); }); - it("rejects a bad signature before parsing the body, without an event", async () => { + it("rejects a bad signature before parsing the body", async () => { const parse = vi.spyOn(JSON, "parse"); const res = await app.fetch(webhook("{not json", { signature: sign("{not json", "wrong") })); expect(res.status).toBe(401); expect(parse).not.toHaveBeenCalled(); parse.mockRestore(); expect(startTask).not.toHaveBeenCalled(); - expect(emit).not.toHaveBeenCalled(); }); it("rejects webhook bodies over 256 KB before starting work", async () => { @@ -80,23 +69,18 @@ describe("gateway", () => { expect(task).toBe("triage"); expect(key).toBe("delivery-1"); expect(TriageInput.safeParse(input).success).toBe(true); - - const event = deliveryEvent(); - expect(event).toMatchObject({ traceId: "trn-123", runId: "trn-123", repo: REPO, subject: "issue#12" }); - expect(event.payload).toMatchObject({ - event: "issues", - action: "labeled", - outcome: "started", - deliveryId: "delivery-1", - }); }); - it("returns 502 when startTask fails so GitHub records a failed delivery", async () => { + it("returns 502 and logs why when startTask fails, so GitHub records a failed delivery", async () => { + const warn = vi.spyOn(console, "error").mockImplementation(() => {}); startTask.mockRejectedValueOnce(new Error("render down")); const res = await app.fetch(webhook(JSON.stringify(triagePayload))); expect(res.status).toBe(502); - const event = deliveryEvent(); - expect(event.traceId).toBe("delivery-1"); - expect(event.payload).toMatchObject({ outcome: "start failed", reason: "render down" }); + expect(JSON.parse(warn.mock.calls[0]?.[0] as string)).toMatchObject({ + msg: "could not start task", + deliveryId: "delivery-1", + error: "render down", + }); + warn.mockRestore(); }); }); diff --git a/tests/pr-review.test.ts b/tests/pr-review.test.ts index 34159f5..7dccfc8 100644 --- a/tests/pr-review.test.ts +++ b/tests/pr-review.test.ts @@ -3,7 +3,7 @@ import type { FactoryEvent } from "../app/events/index.js"; const events: FactoryEvent[] = []; vi.mock("../app/events/index.js", () => ({ - emit: async (event: FactoryEvent) => { + emit: (event: FactoryEvent) => { events.push(event); }, })); diff --git a/tests/runner.test.ts b/tests/runner.test.ts index 96b21b7..b59cc80 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -84,6 +84,19 @@ describe("submit", { timeout: 30_000 }, () => { expect(terminal(events)).toMatchObject({ type: "result", output: GOOD, turns: 2 }); expect(code).toBe(0); }); + + it("stops after the last rejected attempt and says what was wrong with it", async () => { + const ws = makeWorkspace(); + const bad = call(SUBMIT, { summary: "ok summary", files: "README.md" }); + const { code, events } = await runWithTestStub(makeInput(ws), () => [bad, bad, bad, call(SUBMIT, GOOD)]); + const end = terminal(events); + expect(end).toMatchObject({ type: "error", code: "invalid_submission" }); + const message = end.type === "error" ? end.message : ""; + expect(message).toContain("rejected 3 times; last: /files"); + expect(message).toContain("submitted fields: summary, files"); + expect(message).not.toContain("ok summary"); + expect(code).toBe(1); + }); }); describe("run limits", { timeout: 30_000 }, () => { diff --git a/tests/schemas.test.ts b/tests/schemas.test.ts index f49be48..4860a58 100644 --- a/tests/schemas.test.ts +++ b/tests/schemas.test.ts @@ -39,6 +39,14 @@ describe("TriageDecision", () => { ]); expect(TriageDecision.safeParse({ ...base, decision: "decline", questions: [] }).success).toBe(true); }); + + it("rejects a misspelled field by name, so the agent can fix it", () => { + const { relevantPaths, ...rest } = base; + const misspelled = { ...rest, decision: "draft", questions: [], relevant_paths: relevantPaths }; + expect(issues(TriageDecision.safeParse(misspelled))).toEqual( + expect.arrayContaining(["relevantPaths: Invalid input: expected array, received undefined", ': Unrecognized key: "relevant_paths"']), + ); + }); }); describe("Spec", () => { diff --git a/tests/scripts-doctor.test.ts b/tests/scripts-doctor.test.ts index 6fd978f..d8f58d1 100644 --- a/tests/scripts-doctor.test.ts +++ b/tests/scripts-doctor.test.ts @@ -1,6 +1,6 @@ import { generateKeyPairSync } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { type DbClient, type DoctorOptions, runDoctor } from "../scripts/lib/doctor.js"; +import { type DoctorOptions, runDoctor } from "../scripts/lib/doctor.js"; import { type CheckResult, Reporter } from "../scripts/lib/output.js"; const { privateKey } = generateKeyPairSync("rsa", { @@ -12,7 +12,6 @@ const PRIVATE_KEY_B64 = Buffer.from(privateKey).toString("base64"); const TOKEN = "ghs_installationTOKEN123"; const SECRETS = { - DATABASE_URL: "postgresql://factory:pgSECRETpw@db.internal:5432/factory", GITHUB_WEBHOOK_SECRET: "whsec-super-secret-value", RENDER_API_KEY: "rnd_renderSECRETkey", ANTHROPIC_API_KEY: "sk-ant-SECRET-model-key", @@ -94,36 +93,8 @@ function fakeFetch(world: World = {}) { return { fetch, calls }; } -function fakeDb(opts: { events?: boolean; migrations?: string[] | null; unreachable?: boolean } = {}) { - const queries: string[] = []; - const connectDb = async (): Promise => { - if (opts.unreachable) throw new Error("connect ECONNREFUSED 10.0.0.1:5432"); - return { - async query(sql: string) { - queries.push(sql); - if (sql.includes("to_regclass")) { - return { - rows: [ - { - events: opts.events === false ? null : "events", - migrations: opts.migrations === null ? null : "schema_migrations", - }, - ] as T[], - }; - } - if (sql.includes("from schema_migrations")) { - return { rows: (opts.migrations ?? ["0001_events.sql"]).map((filename) => ({ filename })) as T[] }; - } - throw new Error(`unexpected query: ${sql}`); - }, - async end() {}, - }; - }; - return { connectDb, queries }; -} - -async function doctor(overrides: Partial & { world?: World; db?: ReturnType } = {}) { - const { world, db, ...rest } = overrides; +async function doctor(overrides: Partial & { world?: World } = {}) { + const { world, ...rest } = overrides; const fake = fakeFetch(world); const lines: string[] = []; const out = new Reporter((line) => lines.push(line)); @@ -135,8 +106,6 @@ async function doctor(overrides: Partial & { world?: World; db?: services: ["gateway", "workflows"], repos: ["acme/web"], models: { medium: "anthropic/sonnet", large: "anthropic/opus" }, - migrations: ["0001_events.sql"], - connectDb: (db ?? fakeDb()).connectDb, ...rest, }, out, @@ -147,7 +116,7 @@ async function doctor(overrides: Partial & { world?: World; db?: } function expectNoSecrets(output: string) { - for (const value of [...Object.values(SECRETS), TOKEN, "pgSECRETpw", privateKey.split("\n")[1] ?? ""]) { + for (const value of [...Object.values(SECRETS), TOKEN, privateKey.split("\n")[1] ?? ""]) { expect(output).not.toContain(value); } } @@ -179,7 +148,6 @@ describe("doctor", () => { labels: [], tasksStatus: 500, }, - db: fakeDb({ unreachable: true }), }); expectNoSecrets(output); }); diff --git a/tests/spec-to-pr.test.ts b/tests/spec-to-pr.test.ts index f837a84..4c425f5 100644 --- a/tests/spec-to-pr.test.ts +++ b/tests/spec-to-pr.test.ts @@ -3,7 +3,7 @@ import type { FactoryEvent } from "../app/events/index.js"; const events: FactoryEvent[] = []; vi.mock("../app/events/index.js", () => ({ - emit: async (event: FactoryEvent) => { + emit: (event: FactoryEvent) => { events.push(event); }, })); diff --git a/tests/triage.test.ts b/tests/triage.test.ts index 2c1374e..edf0cec 100644 --- a/tests/triage.test.ts +++ b/tests/triage.test.ts @@ -3,7 +3,7 @@ import type { FactoryEvent } from "../app/events/index.js"; const events: FactoryEvent[] = []; vi.mock("../app/events/index.js", () => ({ - emit: async (event: FactoryEvent) => { + emit: (event: FactoryEvent) => { events.push(event); }, }));