diff --git a/apps/cli/docs/supabase/db/pull.md b/apps/cli/docs/supabase/db/pull.md index 1e8ab966f5..957adb3365 100644 --- a/apps/cli/docs/supabase/db/pull.md +++ b/apps/cli/docs/supabase/db/pull.md @@ -19,16 +19,16 @@ On a project that falls back to the legacy migra engine, don't use `--declarativ 1. Delete the files the export wrote to the declarative directory (`supabase/schemas/` by default). Files that were already there are left in place. 2. Restore `[db.migrations].schema_paths` in `config.toml` to its previous value, or remove it if the export added it. -pg-delta plans are execution-aware: when a plan crosses a transaction boundary — for example `ALTER TYPE ... ADD VALUE` followed by a statement that uses the new enum value, which cannot run in the same transaction — `db pull` writes one ordered migration file per plan unit instead of a single file (for example `_remote_schema_schema_changes.sql` and `_remote_schema_after_enum_values.sql`), each recorded in the migration history. The common case (a single unit) still produces exactly one `_remote_schema.sql` file. +pg-delta plans are execution-aware: when a plan crosses a transaction boundary — for example `ALTER TYPE ... ADD VALUE` followed by a statement that uses the new enum value, which cannot run in the same transaction — `db pull` writes one ordered migration file per plan unit instead of a single file (for example `_remote_schema_1.sql` and `_remote_schema_2.sql`), each recorded in the migration history. Units whose statements cannot run inside a transaction start with a `-- pg-delta: transaction=false` directive that the CLI apply paths honor. The common case (a single unit) still produces exactly one `_remote_schema.sql` file. By default the emitted SQL is formatted with the same settings the declarative export uses (uppercase keywords, wrapped at a max width of 180, indented and column-aligned). Configure overrides with `[experimental.pgdelta] format_options` in `config.toml`, or set `format_options = "null"` to opt out and emit raw, unformatted statements. pg-delta is the default diff engine: the migration-file `db pull` workflow uses pg-delta for the shadow diff step unless configured otherwise; it does not switch to declarative output. With pg-delta, `[db.migrations] schema_paths` does not affect this workflow (migra still reads it, for local targets only); use `supabase db schema declarative sync` for declarative schema files. To fall back to the legacy migra engine, set `enabled = false` under `[experimental.pgdelta]` in `config.toml`, or pass `--diff-engine migra` for a single run. -Under the pg-delta engine, the pull captures your own customizations in Supabase-managed schemas automatically. Examples are triggers on managed tables like `auth.users` and RLS policies on `storage.objects` and `storage.buckets`. Under the legacy migra engine, the initial pull's `pg_dump` skips managed schemas. The diff pass that follows appends your trigger and RLS policy changes there to the same migration. If an older CLI reports that `auth` and `storage` were excluded, capture existing customizations there with `--schema auth,storage`. - When pulling with `--remote=`, prefer a direct connection (`db..supabase.co:5432`) over the connection pooler so pg-delta can introspect the full catalog reliably. +Under the pg-delta engine, a plain `db pull` captures your own customizations in Supabase-managed schemas: triggers on managed tables whose function lives outside the managed schemas (for example a trigger on `auth.users` calling a function in `public`), RLS policies on any `auth` table, and RLS policies on `storage.objects`, `storage.buckets`, and `realtime.messages`. Other objects you create inside managed schemas, such as your own functions or indexes, are not diffed. On later pulls, `--schema` restricts the diff to the listed schemas, so a customization in `auth` is only captured when `auth` is listed or `--schema` is omitted; the initial pull ignores `--schema`. Under the legacy migra engine, the initial pull's `pg_dump` skips the managed schemas, and the migra diff pass that follows appends your trigger and RLS policy changes there to the same migration file. If an older CLI reports that `auth` and `storage` were excluded, capture existing customizations there with `--schema auth,storage`. + ## Debugging empty pg-delta pulls If `db pull` reports `No schema changes found` but you expect schema output, set `PGDELTA_DEBUG=1` before running the command. Unlike `--debug`, this keeps SSL enabled for hosted Supabase connections. diff --git a/apps/cli/scripts/detect-unmirrored-images.ts b/apps/cli/scripts/detect-unmirrored-images.ts index ee055e499d..5bf3319f12 100644 --- a/apps/cli/scripts/detect-unmirrored-images.ts +++ b/apps/cli/scripts/detect-unmirrored-images.ts @@ -88,12 +88,13 @@ export interface InspectResult { readonly stderr: string; } -/** Backoff before each retry of a rate-limited (HTTP 429) registry probe. */ +/** Backoff before each retry of a rate-limited (HTTP 429 or OCI `toomanyrequests`) registry probe. */ export const RATE_LIMIT_RETRY_DELAYS_MS: ReadonlyArray = [10_000, 30_000, 60_000]; /** - * Runs a registry probe, retrying with backoff while the registry answers 429; the anonymous - * public.ecr.aws quota is shared per runner IP and usually recovers within a minute. + * Runs a registry probe, retrying with backoff while the registry rate-limits it (HTTP 429 or OCI + * `toomanyrequests`); the anonymous public.ecr.aws quota is shared per runner IP and usually + * recovers within a minute. */ export async function inspectWithRateLimitRetry( target: string, @@ -104,7 +105,11 @@ export async function inspectWithRateLimitRetry( for (let attempt = 0; ; attempt++) { const { exit, stderr } = inspect(); const delay = delaysMs[attempt]; - if (exit !== 0 && stderr.includes("429 Too Many Requests") && delay !== undefined) { + if ( + exit !== 0 && + /429 Too Many Requests|toomanyrequests/i.test(stderr) && + delay !== undefined + ) { console.error(`rate limited inspecting ${target}, retrying in ${delay}ms`); await sleep(delay); continue; diff --git a/apps/cli/scripts/detect-unmirrored-images.unit.test.ts b/apps/cli/scripts/detect-unmirrored-images.unit.test.ts index a3c2d93172..07b699f80a 100644 --- a/apps/cli/scripts/detect-unmirrored-images.unit.test.ts +++ b/apps/cli/scripts/detect-unmirrored-images.unit.test.ts @@ -91,7 +91,8 @@ describe("detect unmirrored images", () => { return Promise.resolve(); }; - const responses = [rateLimited, rateLimited, { exit: 0, stderr: "" }]; + const quotaExceeded = { exit: 1, stderr: "ERROR: toomanyrequests: Data limit exceeded\n" }; + const responses = [rateLimited, quotaExceeded, { exit: 0, stderr: "" }]; await expect( inspectWithRateLimitRetry(target, () => responses.shift()!, sleep, [1, 2, 3]), ).resolves.toBe(true); diff --git a/apps/cli/src/commands/branches/create/create.live.test.ts b/apps/cli/src/commands/branches/create/create.live.test.ts index b00bdf0617..7195391884 100644 --- a/apps/cli/src/commands/branches/create/create.live.test.ts +++ b/apps/cli/src/commands/branches/create/create.live.test.ts @@ -3,11 +3,12 @@ import { randomUUID } from "node:crypto"; import { Cause, Effect, Exit, Schema } from "effect"; import { expect } from "vitest"; -import { requireLiveSuccess, throwWithCleanup, test } from "../../../../tests/helpers/live.ts"; +import { throwWithCleanup, test } from "../../../../tests/helpers/live.ts"; import { awaitLiveBranchEffect, awaitLiveBranchRemovedEffect, removeLiveBranchByNameEffect, + runLiveBranchCreateEffect, } from "../../../../tests/helpers/branches-live.ts"; const CreatedBranchRef = Schema.Struct({ project_ref: Schema.String }); @@ -20,16 +21,7 @@ test("creates a preview branch", ({ cliEffect, project }) => let branchRef: string | undefined; const target = Effect.gen(function* () { - const result = yield* cliEffect([ - "branches", - "create", - name, - "--output-format", - "json", - "--yes", - `--remote=${project.ref}`, - ]); - requireLiveSuccess(result, "branches create"); + const result = yield* runLiveBranchCreateEffect(cliEffect, project, name); const refBody = yield* Schema.decodeEffect(Schema.fromJsonString(CreatedBranchRef))( result.stdout, ); diff --git a/apps/cli/src/shared/services/Dockerfile b/apps/cli/src/shared/services/Dockerfile index 74b0373f1d..ca8f2ec08f 100644 --- a/apps/cli/src/shared/services/Dockerfile +++ b/apps/cli/src/shared/services/Dockerfile @@ -35,7 +35,7 @@ FROM supabase/gotrue:v2.197.0 AS gotrue FROM supabase/realtime:v2.147.1 AS realtime -FROM supabase/storage-api:v1.84.19 AS storage +FROM supabase/storage-api:v1.84.20 AS storage FROM supabase/logflare:1.50.15 AS logflare # One-shot job images (not started as long-running containers); differ's alias diff --git a/apps/cli/tests/helpers/branches-live.integration.test.ts b/apps/cli/tests/helpers/branches-live.integration.test.ts index 7bde7ef27e..23d6fb0546 100644 --- a/apps/cli/tests/helpers/branches-live.integration.test.ts +++ b/apps/cli/tests/helpers/branches-live.integration.test.ts @@ -224,6 +224,91 @@ describe("live branch lifecycle helpers", () => { }), ); + it.effect( + "re-issues create once after a platform database restart, clearing the half-made branch", + () => + Effect.gen(function* () { + const state: State = { calls: [], listed: false, getReady: false, deleteAttempts: 0 }; + let creates = 0; + const cli: BranchCli = (args) => + Effect.sync(() => { + state.calls.push(args.join(" ")); + if (args[1] === "create") { + creates += 1; + state.listed = true; + return creates === 1 + ? result( + JSON.stringify({ + _tag: "Error", + error: { + message: + "unexpected create branch status 400: FATAL: 57P01: terminating connection due to administrator command", + }, + }), + "", + 1, + ) + : result(JSON.stringify(branch)); + } + if (args[1] === "delete") { + state.listed = false; + return result(); + } + if (args[1] === "list") + return result( + JSON.stringify(state.listed ? [defaultBranch, branch] : [defaultBranch]), + ); + return result(); + }); + const ref = yield* createLiveBranchEffect(cli, project, branch.name); + expect(ref).toBe(branchRef); + expect(creates).toBe(2); + const deleteIndex = state.calls.indexOf( + `branches delete ${branch.name} --remote=${project.ref} --yes`, + ); + expect(deleteIndex).toBeGreaterThan(-1); + expect( + state.calls.findLastIndex((call) => call.includes("branches create")), + ).toBeGreaterThan(deleteIndex); + }), + ); + + it.effect("fails a create rejected by a second platform database restart", () => + Effect.gen(function* () { + let creates = 0; + const cli: BranchCli = (args) => + Effect.sync(() => { + if (args[1] === "create") { + creates += 1; + return result("", "status 400: the database system is shutting down", 1); + } + return result(JSON.stringify([defaultBranch])); + }); + const exit = yield* Effect.exit(createLiveBranchEffect(cli, project, branch.name)); + if (!Exit.isFailure(exit)) throw new Error("expected create failure"); + expect(String(exit.cause)).toContain("the database system is shutting down"); + expect(creates).toBe(2); + }), + ); + + it.effect("keeps the restart rejection when cleanup before the retry fails", () => + Effect.gen(function* () { + const cli: BranchCli = (args) => + Effect.sync(() => + args[1] === "create" + ? result("", "status 400: FATAL: 57P01: terminating connection", 1) + : result("", "list forbidden", 1), + ); + const exit = yield* Effect.exit(createLiveBranchEffect(cli, project, branch.name)); + if (!Exit.isFailure(exit)) throw new Error("expected create failure"); + const flatten = (error: unknown): string[] => + error instanceof AggregateError ? error.errors.flatMap(flatten) : [String(error)]; + const causes = flatten(Cause.squash(exit.cause)).join("\n"); + expect(causes).toContain("57P01"); + expect(causes).toContain("list forbidden"); + }), + ); + it.effect("never echoes branches get stdout, which prints the branch secrets", () => Effect.gen(function* () { const cli: BranchCli = () => Effect.succeed(result("DB_PASSWORD sentinel-secret", "", 1)); diff --git a/apps/cli/tests/helpers/branches-live.ts b/apps/cli/tests/helpers/branches-live.ts index 6190f51e29..13e84fea23 100644 --- a/apps/cli/tests/helpers/branches-live.ts +++ b/apps/cli/tests/helpers/branches-live.ts @@ -139,19 +139,57 @@ function branchRefFromCreate( ); } +// The platform rejects `branches create` with a 400 when the database it configures restarts mid-create. +const PLATFORM_DATABASE_RESTART = /\b57P01\b|the database system is shutting down/u; + +function isPlatformDatabaseRestart(error: unknown): boolean { + return ( + error instanceof BranchCommandFailed && + PLATFORM_DATABASE_RESTART.test(`${error.stdout ?? ""}\n${error.stderr}`) + ); +} + +/** Runs `branches create`, re-issuing it once when the platform restarted the database mid-create. */ +export function runLiveBranchCreateEffect( + cli: BranchCli, + project: LiveProject, + name: string, +): Effect.Effect, never> { + const create = Effect.suspend(() => + command( + cli, + ["branches", "create", name, `--remote=${project.ref}`, "--output-format", "json", "--yes"], + "branches create", + undefined, + ), + ); + const clearRejected = branchIsListed(cli, project, name, "name").pipe( + Effect.flatMap((listed) => + listed ? removeLiveBranchByNameEffect(cli, project, name) : Effect.succeed(true as const), + ), + ); + return create.pipe( + Effect.catch((rejected) => + isPlatformDatabaseRestart(rejected) + ? clearRejected.pipe( + Effect.mapError( + (cleanup) => + new AggregateError([rejected, cleanup], "Branch create retry cleanup failed"), + ), + Effect.flatMap(() => create), + ) + : Effect.fail(rejected), + ), + ); +} + /** Creates one named branch and captures its immutable reference before polling readiness. */ export function createLiveBranchEffect( cli: BranchCli, project: LiveProject, name: string, ): Effect.Effect, never> { - const created = command( - cli, - ["branches", "create", name, `--remote=${project.ref}`, "--output-format", "json", "--yes"], - "branches create", - undefined, - ); - return created.pipe( + return runLiveBranchCreateEffect(cli, project, name).pipe( Effect.flatMap((result) => branchRefFromCreate(result)), Effect.catch((primary) => removeLiveBranchByNameEffect(cli, project, name).pipe( diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 1f18982d6d..f2c0b8f792 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -241,6 +241,27 @@ const definitions: Readonly> = { }, }, }, + "17.11.0.007-orioledb": { + upstreamVersion: "17.11.0.007-orioledb", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:17.11.0.007-orioledb-r0@sha256:a62869318bce0ad728467c2ff3e15738cfb5d99cb766e052e8534960975afa01", + upstreamImage: "supabase/postgres:17.11.0.007-orioledb", + natives: { + "darwin-arm64": { + archive: "aedf303d33aaafdd109750d4c8435bb799b106b64a9241e03abb301eba37353b", + manifest: "2786921fac103062604afa982d182e169d635775964be94f1dced810e8d01741", + }, + "linux-amd64": { + archive: "534128c4c64693b5625ab2cc17f0c26be98e96d0a98eeb6e8c8d1b7e3805a940", + manifest: "eb3bb0177288d468abf5cb6e86ea0f9af69b165e72e3f6244dbd2551d87eca13", + }, + "linux-arm64": { + archive: "7122ddc908d48833c1c8f4410921a634ff98d86ac61b1d88d996bb053ef168e3", + manifest: "99ded1c4adf9a1a8ed3798463abad775e07f0231b5f421706bc2ce42290c9cc5", + }, + }, + }, }, ), rest: definition( @@ -322,23 +343,23 @@ const definitions: Readonly> = { storage: definition( "storage", { - upstreamVersion: "v1.84.19", + upstreamVersion: "v1.84.20", revision: 0, image: - "ghcr.io/supabase/cli/storage:v1.84.19-r0@sha256:ac022dd6fba7ef5f84d676bbe776445c5e1955b786fe14b75a5669015ea771f2", - upstreamImage: "supabase/storage-api:v1.84.19", + "ghcr.io/supabase/cli/storage:v1.84.20-r0@sha256:d876648995c45980642f9405590c97085ea7144838b6362b83a7b79a9527cdf7", + upstreamImage: "supabase/storage-api:v1.84.20", natives: { "darwin-arm64": { - archive: "019aa9887ffeddf03a1465ae3a59eaaa983a09aecd75b1ad1371aa40cd72f2ea", - manifest: "73395585a635aad109912ae78206f68939c7c927cc2b69db9923f15a7804dfa2", + archive: "681201b513e061bd705a61d71360441c75734cd3036a5595347074c5edb137e0", + manifest: "0b61e7005015b0fe1197c84e3501721df412296e9205200c12fe3bc11fe151b2", }, "linux-amd64": { - archive: "61c049488a4deec67a8b4da947057924391f7efa0e7717b0648ca170255ee998", - manifest: "77f281f5684dd16df93162fb38740ade98464295902247403f2f8607bd5d9320", + archive: "59bd88da452b495f8f518bc316ad3b0c86a21228fa2f8d7827e3b0155f35b2b4", + manifest: "1c70eb3e72a0fb280fb0974c09cc5b844e88eae75d570dacbcb2bf3a09579d2e", }, "linux-arm64": { - archive: "689340dcd56b740335a144bf9b3ddb1146e3f13973d0a593df565d22e8ce151a", - manifest: "7d9e3b1d4aa8f91c2707641b64d44d8efe32a61d2923e0c0940729193199fef9", + archive: "d78f253d96d7bb9fb3965b61c38d94b1f525a9f727f5856dc561a64619643055", + manifest: "57b255ae94208a91b36859b395b4b5fc154e3edaf1a14e8f059be837069ab85c", }, }, },