Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ packages/stack/src/functions/generated/* text eol=lf linguist-generated=true
apps/cli/src/shared/feedback/database.types.ts linguist-generated=true
apps/docs/public/cli/config.schema.json linguist-generated=true
apps/docs/public/cli/project-config.schema.json linguist-generated=true
apps/docs/public/cli/supabase-lock/v1.schema.json linguist-generated=true

# Lockfiles. Collapsed rather than hidden -- a dependency change is still one
# click away in review, which is the same treatment GitHub gives package-lock.json.
Expand Down
17 changes: 10 additions & 7 deletions .github/scripts/slim-mirror-payload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@
* Untrusted `mirror-slim-image` dispatch fields. Source and destination URLs
* are derived here; payload strings are only accepted when they match.
*/
import {
NATIVE_TARGETS,
parseSha256Sums,
} from "../../packages/stack/src/preparation/SlimRelease.ts";

const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/;
const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/;
export const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;

export const SOURCE_REGISTRY = "ghcr.io/supabase/cli";
export const DEST_REGISTRY = "public.ecr.aws/supabase/cli";
const NATIVE_TARGETS = ["linux-arm64", "linux-amd64", "darwin-arm64"] as const;

export class InvalidPayloadError extends Error {}

Expand Down Expand Up @@ -209,12 +212,12 @@ export const nativeTripletDigests = (rawManifest: string): NativeFiles | undefin
};
};

export const checksumFor = (contents: string, archiveName: string): string | undefined =>
contents
.split(/\r?\n/u)
.map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu))
.find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]
?.toLowerCase();
/** The lowercase digest `archiveName` has in a `SHA256SUMS` body; throws on a malformed body. */
export const checksumFor = (contents: string, archiveName: string): string | undefined => {
const parsed = parseSha256Sums(contents);
if (parsed.kind === "malformed") throw new InvalidPayloadError(parsed.message);
return parsed.sums.get(archiveName);
};

export const manifestMatches = (
manifestJson: string,
Expand Down
36 changes: 26 additions & 10 deletions .github/scripts/sync-artifacts-catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,6 @@ const manifestWithUpstreamImage =
async () =>
JSON.stringify({ upstream_image: image });

/** A `fetchProvenance` returning a mirrored service's `source`. */
const provenanceWithSource =
(source: string): RevisionIo["fetchProvenance"] =>
async () =>
JSON.stringify({ source });

const unusedFetchManifest: RevisionIo["fetchManifest"] = async () => {
throw new Error("fetchManifest should not be called for this service");
};
Expand Down Expand Up @@ -1510,22 +1504,44 @@ describe("refreshCatalogPin backfills upstreamImage", () => {
);
});

test("resolves a mirrored service's upstreamImage from its oci-provenance source", async () => {
const mirroredIo = (provenanceLine: (provenance: string) => string | undefined): RevisionIo => {
const digests = nativeDigests("ag");
const io: RevisionIo = {
const provenance = JSON.stringify({ source: "docker.io/timberio/vector:0.53.0-alpine" });
const line = provenanceLine(provenance);
return {
listReleaseTags: async () => ["vector-0.53.0-r0"],
fetchChecksums: async () => checksumsFor("vector", "0.53.0-r0", digests),
fetchChecksums: async () =>
[checksumsFor("vector", "0.53.0-r0", digests), line]
.filter((entry) => entry !== undefined)
.join("\n"),
imageDigest: async () => digest("ah"),
s3Sha256: matchingS3("vector", "0.53.0-r0", digests),
fetchManifest: unusedFetchManifest,
fetchProvenance: provenanceWithSource("docker.io/timberio/vector:0.53.0-alpine"),
fetchProvenance: async () => provenance,
};
};

test("resolves a mirrored service's upstreamImage from its oci-provenance source", async () => {
const io = mirroredIo((provenance) => `${hex(provenance)} vector-0.53.0.oci-provenance.json`);

const result = await refreshCatalogPin({ catalog: vectorFixture, service: "vector", io });

expect(result.source).toContain('upstreamImage: "timberio/vector:0.53.0-alpine"');
});

test.each([
["has no SHA256SUMS line", () => undefined, /has no line for vector-0\.53\.0\.oci-provenance/],
[
"does not match SHA256SUMS",
() => `${hex("other")} vector-0.53.0.oci-provenance.json`,
/does not match SHA256SUMS/,
],
])("rejects an oci-provenance file that %s", async (_name, line, message) => {
await expect(
refreshCatalogPin({ catalog: vectorFixture, service: "vector", io: mirroredIo(line) }),
).rejects.toThrow(message);
});

test("rejects a manifest upstream_image carrying a quote or template expression, writing nothing", async () => {
const digests = nativeDigests("ak");
const io: RevisionIo = {
Expand Down
68 changes: 35 additions & 33 deletions .github/scripts/sync-artifacts-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,19 @@
* `docs/adr/0026-slim-artifact-mirrors.md` for the catalog model and this script's three modes.
*/

import { createHash } from "node:crypto";
import {
MIRROR_PRODUCERS,
NATIVE_TARGETS,
UPSTREAM_IMAGE_PATTERN,
normalizeUpstreamImage,
parseSha256Sums,
type SlimNativeTarget,
} from "../../packages/stack/src/preparation/SlimRelease.ts";
import {
DIGEST_PATTERN,
InvalidPayloadError,
SOURCE_REGISTRY,
checksumFor,
escapeRegExp,
nativeFileNames,
nativeObjectUrl,
Expand All @@ -17,9 +25,7 @@ export const CATALOG_PATH = "packages/stack/src/Artifacts.ts";

const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`;

/** Native targets the catalog pins per revision, in `Artifacts.ts`'s serialization order. */
const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const;
type NativeTargetName = (typeof NATIVE_TARGETS)[number];
type NativeTargetName = SlimNativeTarget;

/** Overridable so an integration test can point at a local fixture server instead of GitHub. */
const RELEASES_API =
Expand Down Expand Up @@ -308,11 +314,15 @@ export async function resolveRevisionPin(
};
}

const parsedSums = parseSha256Sums(checksums);
if (parsedSums.kind === "malformed") {
throw new InvalidPayloadError(`${service}-${releaseVersion}: ${parsedSums.message}.`);
}
const natives: Record<string, { archive: string; manifest: string }> = {};
for (const target of NATIVE_TARGETS) {
const files = nativeFileNames(service, releaseVersion, target);
const archive = checksumFor(checksums, files.archive);
const manifest = checksumFor(checksums, files.manifest);
const archive = parsedSums.sums.get(files.archive);
const manifest = parsedSums.sums.get(files.manifest);
if (archive === undefined || manifest === undefined) {
return {
status: "incomplete",
Expand Down Expand Up @@ -384,14 +394,6 @@ export async function resolveRevisionPin(
};
}

/**
* Services slim-services mirrors from an unmodified upstream image rather than building from
* source: their `upstreamImage` comes from the release's `oci-provenance.json` `source` field,
* not a native target's manifest. Keep this in sync with `SlimServicesSource`'s mirror-mode
* services.
*/
const MIRROR_MODE_SOURCE_SERVICES: ReadonlySet<string> = new Set(["vector", "mailpit", "imgproxy"]);

function parseJsonRecord(raw: string): Record<string, unknown> | undefined {
let parsed: unknown;
try {
Expand All @@ -409,26 +411,9 @@ function stringField(record: Record<string, unknown> | undefined, key: string):
return typeof value === "string" && value.length > 0 ? value : undefined;
}

/** Strips a `docker.io/` prefix and any `@sha256:…` digest, to match the Dockerfile's `FROM` form. */
function normalizeUpstreamImage(image: string): string {
const withoutDigest = image.split("@")[0] ?? image;
return withoutDigest.startsWith("docker.io/")
? withoutDigest.slice("docker.io/".length)
: withoutDigest;
}

/**
* A normalized `upstreamImage`: one or more lowercase `registry`/`repository` path segments
* (each `[a-z0-9]`, optionally separated internally by `.`/`_`/`-`), a `:`, and a tag matching
* Docker's own tag grammar. Anchored, so no whitespace, quote, or template/expression syntax can
* slip through — this value gets embedded as a TypeScript string literal in `Artifacts.ts`.
*/
const IMAGE_REFERENCE_PATTERN =
/^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*:[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$/;

/** Validates a normalized `upstreamImage` before it's ever written to `Artifacts.ts`. */
function validateUpstreamImage(image: string, context: string): string {
if (!IMAGE_REFERENCE_PATTERN.test(image)) {
if (!UPSTREAM_IMAGE_PATTERN.test(image)) {
throw new InvalidPayloadError(
`${context} has an invalid upstreamImage ${JSON.stringify(image)}.`,
);
Expand All @@ -450,11 +435,28 @@ async function resolveUpstreamImage(
upstreamVersion: string,
io: RevisionIo,
): Promise<string> {
if (MIRROR_MODE_SOURCE_SERVICES.has(service)) {
if (MIRROR_PRODUCERS.has(service)) {
const provenance = await io.fetchProvenance(service, releaseVersion, upstreamVersion);
if (provenance === undefined) {
throw new InvalidPayloadError(`${service}-${releaseVersion} has no oci-provenance asset.`);
}
const name = `${service}-${upstreamVersion}.oci-provenance.json`;
const checksums = await io.fetchChecksums(service, releaseVersion);
const parsed = parseSha256Sums(checksums ?? "");
if (parsed.kind === "malformed") {
throw new InvalidPayloadError(`${service}-${releaseVersion}: ${parsed.message}.`);
}
const expected = parsed.sums.get(name);
if (expected === undefined) {
throw new InvalidPayloadError(
`${service}-${releaseVersion} SHA256SUMS has no line for ${name}.`,
);
}
if (createHash("sha256").update(provenance).digest("hex") !== expected) {
throw new InvalidPayloadError(
`${service}-${releaseVersion} ${name} does not match SHA256SUMS.`,
);
}
const source = stringField(parseJsonRecord(provenance), "source");
if (source === undefined) {
throw new InvalidPayloadError(
Expand Down
4 changes: 4 additions & 0 deletions .oxlintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@
"packages/stack/src/effect.integration.test.ts",
"packages/stack/src/port-reservation.e2e.test.ts",
"packages/stack/src/host/CommandRunner.ts",
"packages/stack/src/internal/schema-writers.ts",
"packages/stack/src/preparation/ArtifactStore.ts",
"packages/stack/src/preparation/artifacts.integration.test.ts",
"packages/stack/src/runtime/Container.ts",
Expand Down Expand Up @@ -402,6 +403,8 @@
// - NativeProcess.ts, native-process.integration.test.ts: need Environment.
// - postgres-user.ts: needs the no-follow lstatPath.
// - DockerDatabaseStorage.ts: needs Publication.
// - schema-writers.ts: needs namespace Lock and Publication; the facade reaches it through
// Registry, Catalog and Database.
"files": [
"packages/stack/src/Network.ts",
"packages/stack/src/Network.integration.test.ts",
Expand All @@ -411,6 +414,7 @@
"packages/stack/src/Owner.integration.test.ts",
"packages/stack/src/effect.integration.test.ts",
"packages/stack/src/port-reservation.e2e.test.ts",
"packages/stack/src/internal/schema-writers.ts",
"packages/stack/src/preparation/ArtifactStore.ts",
"packages/stack/src/preparation/artifacts.integration.test.ts",
"packages/stack/src/runtime/Container.ts",
Expand Down
Loading
Loading