Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,8 @@ to 128 MB files and expire snapshots older than 7 days, keeping at least 5.
Pipelines ingestion is free. Delivery to Iceberg costs $0.06/GB after 50 GB a
month, and R2 storage is billed as usual.

`pnpm db` opens DuckDB with `tools/ops/src/telemetry/duckdb-init.sql`. It attaches the
`pnpm db` opens the DuckDB release pinned in `mise.toml` with
`tools/ops/src/telemetry/duckdb-init.sql` and the views in `duckdb-views.sql`. It attaches the
spans catalog with a read-only `R2_DATA_READ_TOKEN`, which it takes from
Infisical when the variable is unset. `pnpm db --preview` reads the preview
spans catalog, and DuckDB arguments pass
Expand Down
1 change: 1 addition & 0 deletions mise.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
[tools]
node = "26.13.0"
"aqua:pnpm/pnpm" = "12.16.1"
duckdb = "1.5.5"

[settings]
# Keep the selected tools ahead of inherited global launchers.
Expand Down
30 changes: 26 additions & 4 deletions tools/ops/src/telemetry/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,35 @@ import { optionalEnv } from "../outputs";
import { opsError } from "../process";

const INFISICAL_PROJECT_ID = "bb3fe729-e290-4740-84d4-304752f0d719";
export const INIT_SQL = new URL("./duckdb-init.sql", import.meta.url).pathname;
const ACCOUNT_ID = "75f9660f39e4dafe8b95980b87e7399a";
export const INIT_SQL = decodeURIComponent(new URL("./duckdb-init.sql", import.meta.url).pathname);
export const VIEWS_SQL = decodeURIComponent(
new URL("./duckdb-views.sql", import.meta.url).pathname,
);

/**
* Attaches the selected span catalog. DuckDB takes an ATTACH path only as a literal, so it runs
* between the init file, which creates the secret, and the views that read `telemetry`.
*/
function attachSpans(environment: "preview" | "production") {
const bucket = `sketchi-telemetry-${environment}`;
return `ATTACH '${ACCOUNT_ID}_${bucket}' AS telemetry (TYPE ICEBERG, ENDPOINT 'https://catalog.cloudflarestorage.com/${ACCOUNT_ID}/${bucket}', SECRET r2_data_read, READ_ONLY)`;
}

/** The DuckDB invocation for `pnpm db <args>`: direct with a token, through Infisical without. */
export function duckdbCommand(args: readonly string[], hasToken: boolean) {
const preview = args.includes("--preview");
const duckdbArgs = ["-init", INIT_SQL, ...args.filter((arg) => arg !== "--preview")];
const env = { TELEMETRY_ENV: preview ? "preview" : "production" };
const environment = args.includes("--preview") ? "preview" : "production";
const duckdbArgs = [
"-init",
INIT_SQL,
"-cmd",
attachSpans(environment),
"-cmd",
// The DuckDB shell splits a dot command's arguments at spaces outside double quotes.
`.read "${VIEWS_SQL}"`,
...args.filter((arg) => arg !== "--preview"),
];
const env = { TELEMETRY_ENV: environment };
if (hasToken) return { command: "duckdb", args: duckdbArgs, env };
return {
command: "infisical",
Expand Down
Loading
Loading