diff --git a/.changeset/d1-retry-outage-budget.md b/.changeset/d1-retry-outage-budget.md new file mode 100644 index 0000000000..0825d62fe6 --- /dev/null +++ b/.changeset/d1-retry-outage-budget.md @@ -0,0 +1,5 @@ +--- +"executor": patch +--- + +Reads and unreached writes to D1 are retried for up to 20 seconds instead of five attempts in under four seconds, so a D1 outage of about 15 seconds no longer fails `search` and `invoke` with `Internal tool error`. diff --git a/apps/host-cloudflare/src/db/d1-retry.test.ts b/apps/host-cloudflare/src/db/d1-retry.test.ts index c1fb5f5573..ae13275b3d 100644 --- a/apps/host-cloudflare/src/db/d1-retry.test.ts +++ b/apps/host-cloudflare/src/db/d1-retry.test.ts @@ -114,17 +114,39 @@ describe("retryTransientD1Call", () => { }), ); - it.effect("gives up after five attempts and surfaces the original error", () => + it.effect("rides out a 16 second outage of a read", () => Effect.gen(function* () { - const { call, calls } = failingCall( - [INCIDENT_ERROR, INCIDENT_ERROR, INCIDENT_ERROR, INCIDENT_ERROR, INCIDENT_ERROR], - "rows", - ); + const outage = { down: true }; + const call = (): Promise => + outage.down + ? // oxlint-disable-next-line executor/no-promise-reject -- test double: a D1 call that rejects like the binding does + Promise.reject(bindingRejection(INCIDENT_ERROR)) + : Promise.resolve("rows"); + + const fiber = yield* Effect.forkChild(Effect.exit(retryTransientD1Call("all", true, call))); + yield* TestClock.adjust(Duration.seconds(16)); + outage.down = false; + yield* TestClock.adjust(Duration.seconds(4)); + const exit = yield* Fiber.join(fiber); + + expect(exit).toEqual(Exit.succeed("rows")); + }), + ); + + it.effect("gives up once the retry budget is spent and surfaces the original error", () => + Effect.gen(function* () { + const calls = { count: 0 }; + const call = (): Promise => { + calls.count += 1; + // oxlint-disable-next-line executor/no-promise-reject -- test double: a D1 call that rejects like the binding does + return Promise.reject(bindingRejection(INCIDENT_ERROR)); + }; const exit = yield* runWithVirtualTime(retryTransientD1Call("all", true, call)); expect(exit).toEqual(Exit.fail(new D1CallError({ cause: bindingRejection(INCIDENT_ERROR) }))); - expect(calls.count).toBe(5); + expect(calls.count).toBeGreaterThan(5); + expect(calls.count).toBeLessThan(25); }), ); diff --git a/apps/host-cloudflare/src/db/d1-retry.ts b/apps/host-cloudflare/src/db/d1-retry.ts index acd8c84da2..9140c4535b 100644 --- a/apps/host-cloudflare/src/db/d1-retry.ts +++ b/apps/host-cloudflare/src/db/d1-retry.ts @@ -80,16 +80,18 @@ export const isRetryableD1Failure = (failure: unknown, readOnly: boolean): boole return failureClass === "maybe-executed" && readOnly; }; -export const D1_RETRY_MAX_ATTEMPTS = 5; +export const D1_RETRY_BUDGET = Duration.seconds(20); const D1_RETRY_BASE_DELAY = Duration.millis(200); const D1_RETRY_MAX_DELAY = Duration.seconds(2); -/** 200ms, 400ms, 800ms, 1600ms (each jittered to 80-120%, capped at 2s): - * five attempts within about 3.6 seconds, so an MCP call still answers. */ +/** 200ms, 400ms, 800ms, 1600ms, then 2s (each jittered to 80-120%), for as + * long as the 20 second budget lasts. D1 outages seen in production last + * about 15 seconds, and most failed attempts return in milliseconds, so a + * count-based limit gave up after under 4 seconds, long before D1 recovered. */ export const d1RetrySchedule = Schedule.exponential(D1_RETRY_BASE_DELAY).pipe( Schedule.modifyDelay((_, delay) => Effect.succeed(Duration.min(delay, D1_RETRY_MAX_DELAY))), Schedule.jittered, - Schedule.both(Schedule.recurs(D1_RETRY_MAX_ATTEMPTS - 1)), + Schedule.both(Schedule.during(D1_RETRY_BUDGET)), ); /** A rejection from the D1 binding, kept as-is in `cause`. */ @@ -108,13 +110,12 @@ export const retryTransientD1Call = ( const current = attempt; return Effect.tryPromise({ try: call, catch: (cause) => new D1CallError({ cause }) }).pipe( Effect.tapError(({ cause: failure }) => - current < D1_RETRY_MAX_ATTEMPTS && isRetryableD1Failure(failure, readOnly) + isRetryableD1Failure(failure, readOnly) ? Effect.logWarning("D1 transient failure, retrying").pipe( Effect.annotateLogs({ "d1.operation": operation, "d1.read_only": readOnly, "d1.attempt": current, - "d1.max_attempts": D1_RETRY_MAX_ATTEMPTS, "d1.failure_class": classifyD1Failure(failure), "d1.failure": failureMessages(failure)[0] ?? "unknown", }),