diff --git a/docs/architecture.md b/docs/architecture.md index 744681f3..39da81a3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,7 +176,8 @@ to 128 MB files and expire snapshots older than 7 days, keeping at least 5. Pipelines ingestion is free. Delivery to Iceberg costs $0.06/GB after 50 GB a month, and R2 storage is billed as usual. -`pnpm db` opens DuckDB with `tools/ops/src/telemetry/duckdb-init.sql`. It attaches the +`pnpm db` opens the DuckDB release pinned in `mise.toml` with +`tools/ops/src/telemetry/duckdb-init.sql` and the views in `duckdb-views.sql`. It attaches the spans catalog with a read-only `R2_DATA_READ_TOKEN`, which it takes from Infisical when the variable is unset. `pnpm db --preview` reads the preview spans catalog, and DuckDB arguments pass diff --git a/mise.toml b/mise.toml index 52b453e7..f1bf3a9f 100644 --- a/mise.toml +++ b/mise.toml @@ -2,6 +2,7 @@ [tools] node = "26.13.0" "aqua:pnpm/pnpm" = "12.16.1" +duckdb = "1.5.5" [settings] # Keep the selected tools ahead of inherited global launchers. diff --git a/tools/ops/src/telemetry/db.ts b/tools/ops/src/telemetry/db.ts index 00632fc6..2bda7a21 100644 --- a/tools/ops/src/telemetry/db.ts +++ b/tools/ops/src/telemetry/db.ts @@ -13,13 +13,35 @@ import { optionalEnv } from "../outputs"; import { opsError } from "../process"; const INFISICAL_PROJECT_ID = "bb3fe729-e290-4740-84d4-304752f0d719"; -export const INIT_SQL = new URL("./duckdb-init.sql", import.meta.url).pathname; +const ACCOUNT_ID = "75f9660f39e4dafe8b95980b87e7399a"; +export const INIT_SQL = decodeURIComponent(new URL("./duckdb-init.sql", import.meta.url).pathname); +export const VIEWS_SQL = decodeURIComponent( + new URL("./duckdb-views.sql", import.meta.url).pathname, +); + +/** + * Attaches the selected span catalog. DuckDB takes an ATTACH path only as a literal, so it runs + * between the init file, which creates the secret, and the views that read `telemetry`. + */ +function attachSpans(environment: "preview" | "production") { + const bucket = `sketchi-telemetry-${environment}`; + return `ATTACH '${ACCOUNT_ID}_${bucket}' AS telemetry (TYPE ICEBERG, ENDPOINT 'https://catalog.cloudflarestorage.com/${ACCOUNT_ID}/${bucket}', SECRET r2_data_read, READ_ONLY)`; +} /** The DuckDB invocation for `pnpm db `: direct with a token, through Infisical without. */ export function duckdbCommand(args: readonly string[], hasToken: boolean) { - const preview = args.includes("--preview"); - const duckdbArgs = ["-init", INIT_SQL, ...args.filter((arg) => arg !== "--preview")]; - const env = { TELEMETRY_ENV: preview ? "preview" : "production" }; + const environment = args.includes("--preview") ? "preview" : "production"; + const duckdbArgs = [ + "-init", + INIT_SQL, + "-cmd", + attachSpans(environment), + "-cmd", + // The DuckDB shell splits a dot command's arguments at spaces outside double quotes. + `.read "${VIEWS_SQL}"`, + ...args.filter((arg) => arg !== "--preview"), + ]; + const env = { TELEMETRY_ENV: environment }; if (hasToken) return { command: "duckdb", args: duckdbArgs, env }; return { command: "infisical", diff --git a/tools/ops/src/telemetry/duckdb-init.sql b/tools/ops/src/telemetry/duckdb-init.sql index 17bc51b2..0070c32a 100644 --- a/tools/ops/src/telemetry/duckdb-init.sql +++ b/tools/ops/src/telemetry/duckdb-init.sql @@ -1,349 +1,19 @@ -- Sketchi spans, read from R2 Data Catalog (Iceberg), and the Code Mode usage views --- over them. `pnpm db` runs this with R2_DATA_READ_TOKEN set; `pnpm db --preview` --- reads the preview catalog. See docs/architecture.md, "Persistence and telemetry". +-- over them. `pnpm db` runs this with R2_DATA_READ_TOKEN set, then attaches the selected +-- span catalog as `telemetry` and reads duckdb-views.sql; `pnpm db --preview` selects the +-- preview catalog. See docs/architecture.md, "Persistence and telemetry". INSTALL iceberg; LOAD iceberg; CREATE OR REPLACE SECRET r2_data_read (TYPE ICEBERG, TOKEN getenv('R2_DATA_READ_TOKEN')); SET VARIABLE telemetry_env = coalesce(nullif(getenv('TELEMETRY_ENV'), ''), 'production'); -SET VARIABLE account_id = '75f9660f39e4dafe8b95980b87e7399a'; -SET VARIABLE spans_bucket = 'sketchi-telemetry-' || getvariable('telemetry_env'); -ATTACH getvariable('account_id') || '_' || getvariable('spans_bucket') AS telemetry ( - TYPE ICEBERG, - URI 'https://catalog.cloudflarestorage.com/' || getvariable('account_id') || '/' || getvariable('spans_bucket'), - SECRET r2_data_read -); -- Production Code Mode usage rows written before usage moved into spans, kept as they -- were: codemode_history.sketchi_codemode.usage_events and .usage_issues. -ATTACH getvariable('account_id') || '_sketchi-codemode-usage-analytics-production-v4' AS codemode_history ( +ATTACH '75f9660f39e4dafe8b95980b87e7399a_sketchi-codemode-usage-analytics-production-v4' AS codemode_history ( TYPE ICEBERG, - URI 'https://catalog.cloudflarestorage.com/' || getvariable('account_id') || '/sketchi-codemode-usage-analytics-production-v4', + ENDPOINT 'https://catalog.cloudflarestorage.com/75f9660f39e4dafe8b95980b87e7399a/sketchi-codemode-usage-analytics-production-v4', SECRET r2_data_read, READ_ONLY ); - --- begin: span reassembly --- The rows of spans larger than one stream record, each part once even if a batch --- was delivered twice. -CREATE OR REPLACE VIEW span_parts AS -SELECT DISTINCT ON (trace_id, span_id, part_index) * -FROM telemetry.observability.spans -WHERE part_count > 1 -ORDER BY trace_id, span_id, part_index, __ingest_ts; - --- One row per span. A span larger than one stream record arrives as several rows that --- share its IDs; their attributes, events, error message, error and cause concatenate --- by part_index. part_count > 1 marks a span that was split. A split span appears once --- all its parts have arrived; until then it is in incomplete_spans. -CREATE OR REPLACE VIEW spans AS -WITH - whole AS ( - SELECT * FROM telemetry.observability.spans WHERE part_count = 1 - ), - joined AS ( - SELECT - trace_id, - span_id, - any_value(parent_span_id) AS parent_span_id, - any_value(name) AS name, - any_value(kind) AS kind, - any_value(service) AS service, - any_value(environment) AS environment, - any_value(service_version) AS service_version, - any_value(start_time) AS start_time, - any_value(end_time) AS end_time, - any_value(duration_ms) AS duration_ms, - any_value(outcome) AS outcome, - any_value(error_tag) AS error_tag, - string_agg(error_message, '' ORDER BY part_index) AS error_message, - string_agg(error, '' ORDER BY part_index) AS error, - string_agg(cause, '' ORDER BY part_index) AS cause, - string_agg(attributes, '' ORDER BY part_index) AS attributes, - string_agg(events, '' ORDER BY part_index) AS events, - any_value(ray_id) AS ray_id, - any_value(run_id) AS run_id, - any_value(attempt_id) AS attempt_id, - any_value(part_count) AS part_count, - max(__ingest_ts) AS __ingest_ts - FROM span_parts - GROUP BY trace_id, span_id - HAVING count(DISTINCT part_index) = any_value(part_count) - ), - rows AS ( - SELECT * EXCLUDE (part_index, schema) FROM whole - UNION ALL BY NAME - SELECT * FROM joined - ) -SELECT - trace_id, - span_id, - parent_span_id, - name, - kind, - service, - environment, - service_version, - start_time, - end_time, - duration_ms, - outcome, - error_tag, - error_message, - error::JSON AS error, - cause, - attributes::JSON AS attributes, - events::JSON AS events, - ray_id, - run_id, - attempt_id, - part_count, - __ingest_ts AS ingested_at -FROM rows; - --- Split spans still missing a part: one whose batch is late, or never arrived. -CREATE OR REPLACE VIEW incomplete_spans AS -SELECT - trace_id, - span_id, - any_value(name) AS name, - any_value(service) AS service, - any_value(start_time) AS start_time, - any_value(part_count) AS part_count, - count(DISTINCT part_index) AS parts_received, - list(part_index ORDER BY part_index) AS parts, - max(__ingest_ts) AS last_ingested_at -FROM span_parts -GROUP BY trace_id, span_id -HAVING count(DISTINCT part_index) < any_value(part_count); --- end: span reassembly - --- Every span of one trace, indented under its parent and in start order. The Workers --- clock only advances on I/O, so spans that start in the same millisecond are ordered --- by sketchi.span.start_seq, which counts span starts within one isolate. --- FROM span_tree('') -CREATE OR REPLACE MACRO span_order(start_time, attributes, span_id) AS - strftime(start_time, '%Y-%m-%dT%H:%M:%S.%g') - || lpad(coalesce(attributes->>'sketchi.span.start_seq', '0'), 15, '0') - || span_id; - -CREATE OR REPLACE MACRO span_tree(trace) AS TABLE -WITH RECURSIVE - trace_spans AS MATERIALIZED ( - SELECT * FROM spans WHERE trace_id = trace - ), - tree AS ( - SELECT - s.*, - 0 AS depth, - [span_order(s.start_time, s.attributes, s.span_id)] AS path - FROM trace_spans s - WHERE s.parent_span_id IS NULL - OR s.parent_span_id NOT IN (SELECT span_id FROM trace_spans) - UNION ALL - SELECT - c.*, - t.depth + 1, - list_append(t.path, span_order(c.start_time, c.attributes, c.span_id)) - FROM trace_spans c - JOIN tree t ON c.parent_span_id = t.span_id - ) -SELECT - repeat(' ', depth) || name AS span, - round(duration_ms, 1) AS ms, - outcome, - error_tag, - service, - start_time, - span_id, - parent_span_id, - attributes, - events -FROM tree -ORDER BY path; - --- One row per diagram generation: what was asked, what came back, what it cost. -CREATE OR REPLACE VIEW generations AS -WITH RECURSIVE - runs AS MATERIALIZED ( - SELECT * FROM spans WHERE name = 'diagramGeneration.runWithPolicy' - ), - run_traces AS MATERIALIZED ( - SELECT trace_id, span_id, parent_span_id, name, start_time, duration_ms, outcome, attributes - FROM spans - WHERE trace_id IN (SELECT trace_id FROM runs) - ), - below (run_span_id, trace_id, span_id) AS ( - SELECT span_id, trace_id, span_id FROM runs - UNION ALL - SELECT b.run_span_id, s.trace_id, s.span_id - FROM below b - JOIN run_traces s ON s.trace_id = b.trace_id AND s.parent_span_id = b.span_id - ), - work AS ( - SELECT - b.run_span_id, - count(*) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS model_calls, - sum(s.duration_ms) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS model_ms, - count(*) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt' AND s.outcome <> 'success') AS failed_calls, - count(*) FILTER (s.name IN ('diagramGeneration.repair', 'diagramGeneration.coverage.repair')) AS repairs, - -- LanguageModel.generateText repeats each attempt's usage; count the attempt's. - sum((s.attributes->>'gen_ai.usage.input_tokens')::BIGINT) - FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS input_tokens, - sum((s.attributes->>'gen_ai.usage.output_tokens')::BIGINT) - FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS output_tokens, - arg_min(s.attributes->'gen_ai.system_instructions', s.start_time) - FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS system_instructions, - arg_min(s.attributes->'gen_ai.input.messages', s.start_time) - FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS first_input_messages, - arg_min(s.attributes->'sketchi.output', s.start_time) - FILTER (s.name = 'diagramGeneration.coverage.check') AS coverage_issues, - -- Jev's requests deciding an untyped request's family, before generation. - count(*) FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_rounds, - sum(s.duration_ms) FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_ms, - sum((s.attributes->>'gen_ai.usage.input_tokens')::BIGINT) - FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_input_tokens - FROM below b - JOIN run_traces s ON s.trace_id = b.trace_id AND s.span_id = b.span_id - GROUP BY b.run_span_id - ) -SELECT - r.start_time, - r.trace_id, - r.span_id, - r.service, - r.ray_id, - coalesce(r.run_id, (SELECT max(run_id) FROM spans t WHERE t.trace_id = r.trace_id)) AS run_id, - r.attributes->'sketchi.input'->>'model' AS model, - r.attributes->'sketchi.input'->'prompt'->>'request' AS prompt, - r.attributes->'sketchi.input'->'prompt'->>'requestedType' AS requested_type, - r.attributes->>'family_route' AS family_route, - r.attributes->>'family_narrowing' AS family_narrowing, - r.attributes->>'family_shortlist' AS family_shortlist, - r.attributes->'sketchi.output'->'diagram'->>'type' AS diagram_type, - r.attributes->'sketchi.output'->>'text' AS output, - r.attributes->'sketchi.output'->'diagnostics' AS diagnostics, - r.outcome, - r.error_tag, - r.error_message, - round(r.duration_ms) AS duration_ms, - w.model_calls, - w.failed_calls, - w.repairs, - round(w.model_ms) AS model_ms, - w.input_tokens, - w.output_tokens, - w.input_tokens + w.output_tokens AS total_tokens, - w.coverage_issues, - w.jev_rounds, - round(w.jev_ms) AS jev_ms, - w.jev_input_tokens, - w.system_instructions, - w.first_input_messages -FROM runs r -LEFT JOIN work w ON w.run_span_id = r.span_id -ORDER BY r.start_time DESC; - --- The slowest spans of the last seven days, with the trace to open in span_tree. -CREATE OR REPLACE VIEW slowest_spans AS -SELECT name, service, round(duration_ms) AS ms, outcome, error_tag, start_time, trace_id, span_id -FROM spans -WHERE start_time > (now() AT TIME ZONE 'UTC') - INTERVAL 7 DAY -ORDER BY duration_ms DESC -LIMIT 100; - --- Duration by span name over the last seven days. -CREATE OR REPLACE VIEW span_durations AS -SELECT - name, - service, - count(*) AS spans, - round(median(duration_ms)) AS p50_ms, - round(quantile_cont(duration_ms, 0.95)) AS p95_ms, - round(max(duration_ms)) AS max_ms, - count(*) FILTER (outcome <> 'success') AS not_ok -FROM spans -WHERE start_time > (now() AT TIME ZONE 'UTC') - INTERVAL 7 DAY -GROUP BY ALL -ORDER BY p95_ms DESC; - --- begin: usage views --- One row per Code Mode call (API build, patch, canvas, generate, MCP execute), from the --- usage summary its playground.codeModeUsage.capture span records, with the full --- request and response. Capture spans written before the summary moved onto them have --- none and are left out; their calls are in codemode_history. -CREATE OR REPLACE VIEW usage_events AS -WITH captures AS ( - SELECT *, attributes->'sketchi.usage.event' AS event - FROM spans - WHERE name = 'playground.codeModeUsage.capture' - AND json_exists(attributes, '$."sketchi.usage.event"') -) -SELECT - event->>'event_id' AS event_id, - event->>'run_id' AS run_id, - event->>'attempt_id' AS attempt_id, - start_time AS event_time, - start_time::DATE AS event_date, - event->>'surface' AS surface, - event->>'operation' AS operation, - event->>'status' AS status, - (event->>'status_code')::INTEGER AS status_code, - (event->>'duration_ms')::BIGINT AS duration_ms, - event->>'client' AS client, - event->>'harness' AS harness, - event->>'model' AS model, - event->>'reasoning_level' AS reasoning_level, - event->>'scenario_id' AS scenario_id, - event->>'user_agent' AS user_agent, - (event->>'artifact_count')::INTEGER AS artifact_count, - event->>'artifact_formats' AS artifact_formats, - (event->>'artifact_delivery')::BOOLEAN AS artifact_delivery, - event->'artifacts' AS artifacts, - (event->>'issue_count')::INTEGER AS issue_count, - event->>'issue_codes' AS issue_codes, - event->>'error_message' AS error_message, - event->>'request_path' AS request_path, - event->>'request_method' AS request_method, - attributes->'sketchi.input' AS request, - attributes->'sketchi.output' AS response, - ray_id, - trace_id, - span_id -FROM captures; - --- One row per issue a Code Mode call reported. -CREATE OR REPLACE VIEW usage_issues AS -WITH issues AS ( - SELECT - *, - attributes->'sketchi.usage.event' AS event, - unnest(CAST(attributes->'sketchi.usage.issues' AS JSON[])) AS issue - FROM spans - WHERE name = 'playground.codeModeUsage.capture' - AND json_exists(attributes, '$."sketchi.usage.event"') -) -SELECT - event->>'event_id' AS event_id, - event->>'run_id' AS run_id, - event->>'attempt_id' AS attempt_id, - start_time AS event_time, - start_time::DATE AS event_date, - event->>'surface' AS surface, - event->>'operation' AS operation, - event->>'status' AS status, - issue->>'code' AS issue_code, - issue->>'message' AS issue_message, - issue->>'path' AS issue_path, - issue->>'severity' AS issue_severity, - issue->>'stage' AS issue_stage, - ray_id, - trace_id, - span_id -FROM issues; --- end: usage views - -SELECT - getvariable('telemetry_env') AS environment, - 'spans, incomplete_spans, span_tree(trace_id), generations, slowest_spans, span_durations, usage_events, usage_issues' AS views; diff --git a/tools/ops/src/telemetry/duckdb-views.sql b/tools/ops/src/telemetry/duckdb-views.sql new file mode 100644 index 00000000..5a16f57c --- /dev/null +++ b/tools/ops/src/telemetry/duckdb-views.sql @@ -0,0 +1,323 @@ +-- begin: span reassembly +-- The rows of spans larger than one stream record, each part once even if a batch +-- was delivered twice. +CREATE OR REPLACE VIEW span_parts AS +SELECT DISTINCT ON (trace_id, span_id, part_index) * +FROM telemetry.observability.spans +WHERE part_count > 1 +ORDER BY trace_id, span_id, part_index, __ingest_ts; + +-- One row per span. A span larger than one stream record arrives as several rows that +-- share its IDs; their attributes, events, error message, error and cause concatenate +-- by part_index. part_count > 1 marks a span that was split. A split span appears once +-- all its parts have arrived; until then it is in incomplete_spans. +CREATE OR REPLACE VIEW spans AS +WITH + whole AS ( + SELECT * FROM telemetry.observability.spans WHERE part_count = 1 + ), + joined AS ( + SELECT + trace_id, + span_id, + any_value(parent_span_id) AS parent_span_id, + any_value(name) AS name, + any_value(kind) AS kind, + any_value(service) AS service, + any_value(environment) AS environment, + any_value(service_version) AS service_version, + any_value(start_time) AS start_time, + any_value(end_time) AS end_time, + any_value(duration_ms) AS duration_ms, + any_value(outcome) AS outcome, + any_value(error_tag) AS error_tag, + string_agg(error_message, '' ORDER BY part_index) AS error_message, + string_agg(error, '' ORDER BY part_index) AS error, + string_agg(cause, '' ORDER BY part_index) AS cause, + string_agg(attributes, '' ORDER BY part_index) AS attributes, + string_agg(events, '' ORDER BY part_index) AS events, + any_value(ray_id) AS ray_id, + any_value(run_id) AS run_id, + any_value(attempt_id) AS attempt_id, + any_value(part_count) AS part_count, + max(__ingest_ts) AS __ingest_ts + FROM span_parts + GROUP BY trace_id, span_id + HAVING count(DISTINCT part_index) = any_value(part_count) + ), + rows AS ( + SELECT * EXCLUDE (part_index, schema) FROM whole + UNION ALL BY NAME + SELECT * FROM joined + ) +SELECT + trace_id, + span_id, + parent_span_id, + name, + kind, + service, + environment, + service_version, + start_time, + end_time, + duration_ms, + outcome, + error_tag, + error_message, + error::JSON AS error, + cause, + attributes::JSON AS attributes, + events::JSON AS events, + ray_id, + run_id, + attempt_id, + part_count, + __ingest_ts AS ingested_at +FROM rows; + +-- Split spans still missing a part: one whose batch is late, or never arrived. +CREATE OR REPLACE VIEW incomplete_spans AS +SELECT + trace_id, + span_id, + any_value(name) AS name, + any_value(service) AS service, + any_value(start_time) AS start_time, + any_value(part_count) AS part_count, + count(DISTINCT part_index) AS parts_received, + list(part_index ORDER BY part_index) AS parts, + max(__ingest_ts) AS last_ingested_at +FROM span_parts +GROUP BY trace_id, span_id +HAVING count(DISTINCT part_index) < any_value(part_count); +-- end: span reassembly + +-- Every span of one trace, indented under its parent and in start order. The Workers +-- clock only advances on I/O, so spans that start in the same millisecond are ordered +-- by sketchi.span.start_seq, which counts span starts within one isolate. +-- FROM span_tree('') +CREATE OR REPLACE MACRO span_order(start_time, attributes, span_id) AS + strftime(start_time, '%Y-%m-%dT%H:%M:%S.%g') + || lpad(coalesce(attributes->>'sketchi.span.start_seq', '0'), 15, '0') + || span_id; + +CREATE OR REPLACE MACRO span_tree(trace) AS TABLE +WITH RECURSIVE + trace_spans AS MATERIALIZED ( + SELECT * FROM spans WHERE trace_id = trace + ), + tree AS ( + SELECT + s.*, + 0 AS depth, + [span_order(s.start_time, s.attributes, s.span_id)] AS path + FROM trace_spans s + WHERE s.parent_span_id IS NULL + OR s.parent_span_id NOT IN (SELECT span_id FROM trace_spans) + UNION ALL + SELECT + c.*, + t.depth + 1, + list_append(t.path, span_order(c.start_time, c.attributes, c.span_id)) + FROM trace_spans c + JOIN tree t ON c.parent_span_id = t.span_id + ) +SELECT + repeat(' ', depth) || name AS span, + round(duration_ms, 1) AS ms, + outcome, + error_tag, + service, + start_time, + span_id, + parent_span_id, + attributes, + events +FROM tree +ORDER BY path; + +-- One row per diagram generation: what was asked, what came back, what it cost. +CREATE OR REPLACE VIEW generations AS +WITH RECURSIVE + runs AS MATERIALIZED ( + SELECT * FROM spans WHERE name = 'diagramGeneration.runWithPolicy' + ), + run_traces AS MATERIALIZED ( + SELECT trace_id, span_id, parent_span_id, name, start_time, duration_ms, outcome, attributes + FROM spans + WHERE trace_id IN (SELECT trace_id FROM runs) + ), + below (run_span_id, trace_id, span_id) AS ( + SELECT span_id, trace_id, span_id FROM runs + UNION ALL + SELECT b.run_span_id, s.trace_id, s.span_id + FROM below b + JOIN run_traces s ON s.trace_id = b.trace_id AND s.parent_span_id = b.span_id + ), + work AS ( + SELECT + b.run_span_id, + count(*) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS model_calls, + sum(s.duration_ms) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS model_ms, + count(*) FILTER (s.name = 'diagramGeneration.gatewayChat.attempt' AND s.outcome <> 'success') AS failed_calls, + count(*) FILTER (s.name IN ('diagramGeneration.repair', 'diagramGeneration.coverage.repair')) AS repairs, + -- LanguageModel.generateText repeats each attempt's usage; count the attempt's. + sum((s.attributes->>'gen_ai.usage.input_tokens')::BIGINT) + FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS input_tokens, + sum((s.attributes->>'gen_ai.usage.output_tokens')::BIGINT) + FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS output_tokens, + arg_min(s.attributes->'gen_ai.system_instructions', s.start_time) + FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS system_instructions, + arg_min(s.attributes->'gen_ai.input.messages', s.start_time) + FILTER (s.name = 'diagramGeneration.gatewayChat.attempt') AS first_input_messages, + arg_min(s.attributes->'sketchi.output', s.start_time) + FILTER (s.name = 'diagramGeneration.coverage.check') AS coverage_issues, + -- Jev's requests deciding an untyped request's family, before generation. + count(*) FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_rounds, + sum(s.duration_ms) FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_ms, + sum((s.attributes->>'gen_ai.usage.input_tokens')::BIGINT) + FILTER (s.name = 'diagramGeneration.familyNarrowing.round') AS jev_input_tokens + FROM below b + JOIN run_traces s ON s.trace_id = b.trace_id AND s.span_id = b.span_id + GROUP BY b.run_span_id + ) +SELECT + r.start_time, + r.trace_id, + r.span_id, + r.service, + r.ray_id, + coalesce(r.run_id, (SELECT max(run_id) FROM spans t WHERE t.trace_id = r.trace_id)) AS run_id, + r.attributes->'sketchi.input'->>'model' AS model, + r.attributes->'sketchi.input'->'prompt'->>'request' AS prompt, + r.attributes->'sketchi.input'->'prompt'->>'requestedType' AS requested_type, + r.attributes->>'family_route' AS family_route, + r.attributes->>'family_narrowing' AS family_narrowing, + r.attributes->>'family_shortlist' AS family_shortlist, + r.attributes->'sketchi.output'->'diagram'->>'type' AS diagram_type, + r.attributes->'sketchi.output'->>'text' AS output, + r.attributes->'sketchi.output'->'diagnostics' AS diagnostics, + r.outcome, + r.error_tag, + r.error_message, + round(r.duration_ms) AS duration_ms, + w.model_calls, + w.failed_calls, + w.repairs, + round(w.model_ms) AS model_ms, + w.input_tokens, + w.output_tokens, + w.input_tokens + w.output_tokens AS total_tokens, + w.coverage_issues, + w.jev_rounds, + round(w.jev_ms) AS jev_ms, + w.jev_input_tokens, + w.system_instructions, + w.first_input_messages +FROM runs r +LEFT JOIN work w ON w.run_span_id = r.span_id +ORDER BY r.start_time DESC; + +-- The slowest spans of the last seven days, with the trace to open in span_tree. +CREATE OR REPLACE VIEW slowest_spans AS +SELECT name, service, round(duration_ms) AS ms, outcome, error_tag, start_time, trace_id, span_id +FROM spans +WHERE start_time > (now() AT TIME ZONE 'UTC') - INTERVAL 7 DAY +ORDER BY duration_ms DESC +LIMIT 100; + +-- Duration by span name over the last seven days. +CREATE OR REPLACE VIEW span_durations AS +SELECT + name, + service, + count(*) AS spans, + round(median(duration_ms)) AS p50_ms, + round(quantile_cont(duration_ms, 0.95)) AS p95_ms, + round(max(duration_ms)) AS max_ms, + count(*) FILTER (outcome <> 'success') AS not_ok +FROM spans +WHERE start_time > (now() AT TIME ZONE 'UTC') - INTERVAL 7 DAY +GROUP BY ALL +ORDER BY p95_ms DESC; + +-- begin: usage views +-- One row per Code Mode call (API build, patch, canvas, generate, MCP execute), from the +-- usage summary its playground.codeModeUsage.capture span records, with the full +-- request and response. Capture spans written before the summary moved onto them have +-- none and are left out; their calls are in codemode_history. +CREATE OR REPLACE VIEW usage_events AS +WITH captures AS ( + SELECT *, attributes->'sketchi.usage.event' AS event + FROM spans + WHERE name = 'playground.codeModeUsage.capture' + AND json_exists(attributes, '$."sketchi.usage.event"') +) +SELECT + event->>'event_id' AS event_id, + event->>'run_id' AS run_id, + event->>'attempt_id' AS attempt_id, + start_time AS event_time, + start_time::DATE AS event_date, + event->>'surface' AS surface, + event->>'operation' AS operation, + event->>'status' AS status, + (event->>'status_code')::INTEGER AS status_code, + (event->>'duration_ms')::BIGINT AS duration_ms, + event->>'client' AS client, + event->>'harness' AS harness, + event->>'model' AS model, + event->>'reasoning_level' AS reasoning_level, + event->>'scenario_id' AS scenario_id, + event->>'user_agent' AS user_agent, + (event->>'artifact_count')::INTEGER AS artifact_count, + event->>'artifact_formats' AS artifact_formats, + (event->>'artifact_delivery')::BOOLEAN AS artifact_delivery, + event->'artifacts' AS artifacts, + (event->>'issue_count')::INTEGER AS issue_count, + event->>'issue_codes' AS issue_codes, + event->>'error_message' AS error_message, + event->>'request_path' AS request_path, + event->>'request_method' AS request_method, + attributes->'sketchi.input' AS request, + attributes->'sketchi.output' AS response, + ray_id, + trace_id, + span_id +FROM captures; + +-- One row per issue a Code Mode call reported. +CREATE OR REPLACE VIEW usage_issues AS +WITH issues AS ( + SELECT + *, + attributes->'sketchi.usage.event' AS event, + unnest(CAST(attributes->'sketchi.usage.issues' AS JSON[])) AS issue + FROM spans + WHERE name = 'playground.codeModeUsage.capture' + AND json_exists(attributes, '$."sketchi.usage.event"') +) +SELECT + event->>'event_id' AS event_id, + event->>'run_id' AS run_id, + event->>'attempt_id' AS attempt_id, + start_time AS event_time, + start_time::DATE AS event_date, + event->>'surface' AS surface, + event->>'operation' AS operation, + event->>'status' AS status, + issue->>'code' AS issue_code, + issue->>'message' AS issue_message, + issue->>'path' AS issue_path, + issue->>'severity' AS issue_severity, + issue->>'stage' AS issue_stage, + ray_id, + trace_id, + span_id +FROM issues; +-- end: usage views + +SELECT + getvariable('telemetry_env') AS environment, + 'spans, incomplete_spans, span_tree(trace_id), generations, slowest_spans, span_durations, usage_events, usage_issues' AS views; diff --git a/tools/ops/tests/duckdb-views.test.ts b/tools/ops/tests/duckdb-views.test.ts index 6256a4e3..01f9d0c1 100644 --- a/tools/ops/tests/duckdb-views.test.ts +++ b/tools/ops/tests/duckdb-views.test.ts @@ -1,20 +1,161 @@ -// The checked-in DuckDB views over the span table, run by the real DuckDB CLI on local rows. -import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; +// The checked-in DuckDB init and views, run by the real DuckDB CLI on local rows and a local +// Iceberg REST catalog. +import { execFile, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; import { expect, test } from "vitest"; -import { INIT_SQL } from "../src/telemetry/db"; +import { duckdbCommand, INIT_SQL, VIEWS_SQL } from "../src/telemetry/db"; const init = readFileSync(INIT_SQL, "utf8"); +const views = readFileSync(VIEWS_SQL, "utf8"); const duckdb = spawnSync("duckdb", ["--version"], { encoding: "utf8" }); +const pinnedDuckdb = readFileSync(new URL("../../../mise.toml", import.meta.url), "utf8").match( + /^duckdb = "([\d.]+)"$/m, +)?.[1]; +const CATALOG_HOST = "https://catalog.cloudflarestorage.com"; +const ACCOUNT = "75f9660f39e4dafe8b95980b87e7399a"; +const SPAN_COLUMNS = [ + ...["schema", "trace_id", "span_id", "parent_span_id", "name", "kind", "service"], + ...["environment", "service_version", "start_time", "end_time", "duration_ms", "outcome"], + ...["error_tag", "error_message", "error", "cause", "attributes", "events", "ray_id"], + ...["run_id", "attempt_id", "part_index", "part_count", "__ingest_ts"], +]; +const columnType = (name: string) => + name.endsWith("_time") || name === "__ingest_ts" + ? "timestamp" + : name === "duration_ms" + ? "double" + : name.startsWith("part_") + ? "int" + : "string"; + +/** An Iceberg REST catalog on localhost whose every `observability.spans` table is empty. */ +async function emptySpanCatalog() { + const requests: { path: string; warehouse: string | null; authorization: string | undefined }[] = + []; + const metadata = { + "format-version": 2, + "table-uuid": "00000000-0000-4000-8000-000000000000", + location: "file:///nonexistent/spans", + "last-sequence-number": 0, + "last-updated-ms": 0, + "last-column-id": SPAN_COLUMNS.length, + "current-schema-id": 0, + schemas: [ + { + type: "struct", + "schema-id": 0, + fields: SPAN_COLUMNS.map((name, index) => ({ + id: index + 1, + name, + required: false, + type: columnType(name), + })), + }, + ], + "default-spec-id": 0, + "partition-specs": [{ "spec-id": 0, fields: [] }], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{ "order-id": 0, fields: [] }], + properties: {}, + snapshots: [], + "snapshot-log": [], + "metadata-log": [], + }; + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://catalog"); + requests.push({ + path: url.pathname, + warehouse: url.searchParams.get("warehouse"), + authorization: request.headers.authorization, + }); + response.setHeader("content-type", "application/json"); + if (url.pathname.endsWith("/v1/config")) { + response.end(JSON.stringify({ defaults: {}, overrides: {} })); + } else if (url.pathname.endsWith("/v1/namespaces/observability/tables/spans")) { + response.end( + JSON.stringify({ "metadata-location": `${metadata.location}/v1.json`, metadata }), + ); + } else { + response.statusCode = 404; + response.end(JSON.stringify({ error: { code: 404, message: "absent", type: "NotFound" } })); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + endpoint: `http://127.0.0.1:${port}`, + requests, + close: () => new Promise((resolve) => server.close(resolve)), + }; +} + +test.skipIf(duckdb.status !== 0).each(["preview", "production"] as const)( + "pnpm db opens the %s span catalog and the production usage history with the token", + async (environment) => { + // R2's span manifests need the pinned release: 1.5.2's iceberg extension aborts on them. + expect(duckdb.stdout, "run with the mise.toml DuckDB").toMatch( + new RegExp(`^v${pinnedDuckdb} `), + ); + const catalog = await emptySpanCatalog(); + const directory = mkdtempSync(join(tmpdir(), "sketchi-duckdb-init-")); + try { + const localInit = join(directory, "duckdb-init.sql"); + writeFileSync(localInit, init.replaceAll(CATALOG_HOST, catalog.endpoint)); + const sql = "SELECT getvariable('telemetry_env') AS env, (FROM spans SELECT count(*)) AS n"; + const flags = environment === "preview" ? ["--preview"] : []; + const command = duckdbCommand([...flags, "-noheader", "-list", "-c", sql], true); + const { stdout } = await promisify(execFile)( + command.command, + command.args.map((arg) => + arg === INIT_SQL ? localInit : arg.replaceAll(CATALOG_HOST, catalog.endpoint), + ), + { env: { ...process.env, ...command.env, R2_DATA_READ_TOKEN: "test-token", AI_AGENT: "" } }, + ); + expect(stdout.trim().split("\n").at(-1)).toBe(`${environment}|0`); + const bucket = `sketchi-telemetry-${environment}`; + const history = "sketchi-codemode-usage-analytics-production-v4"; + // Only the selected span catalog opens, beside the production usage history. + expect( + new Set( + catalog.requests + .filter(({ path }) => path.endsWith("/v1/config")) + .map(({ path, warehouse }) => `${path}?warehouse=${warehouse}`), + ), + ).toEqual( + new Set([ + `/${ACCOUNT}/${bucket}/v1/config?warehouse=${ACCOUNT}_${bucket}`, + `/${ACCOUNT}/${history}/v1/config?warehouse=${ACCOUNT}_${history}`, + ]), + ); + expect( + new Set( + catalog.requests.filter(({ path }) => path.includes("/tables/")).map(({ path }) => path), + ), + ).toEqual(new Set([`/${ACCOUNT}/${bucket}/v1/namespaces/observability/tables/spans`])); + expect( + catalog.requests.every(({ authorization }) => authorization === "Bearer test-token"), + ).toBe(true); + } finally { + await catalog.close(); + rmSync(directory, { force: true, recursive: true }); + } + }, +); /** A marked section of the checked-in views, reading a local table instead of the catalog. */ function section(name: string) { - const start = init.indexOf(`-- begin: ${name}`); - const end = init.indexOf(`-- end: ${name}`); - expect(start >= 0 && end > start, `duckdb-init.sql marks its ${name}`).toBe(true); - return init.slice(start, end).replaceAll("telemetry.observability.spans", "raw_spans"); + const start = views.indexOf(`-- begin: ${name}`); + const end = views.indexOf(`-- end: ${name}`); + expect(start >= 0 && end > start, `duckdb-views.sql marks its ${name}`).toBe(true); + return views.slice(start, end).replaceAll("telemetry.observability.spans", "raw_spans"); } const RAW_SPANS = ` diff --git a/tools/ops/tests/telemetry.test.ts b/tools/ops/tests/telemetry.test.ts index 69c0a859..1ebf0c40 100644 --- a/tools/ops/tests/telemetry.test.ts +++ b/tools/ops/tests/telemetry.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { describe, expect, test } from "vitest"; -import { INIT_SQL } from "../src/telemetry/db"; +import { INIT_SQL, VIEWS_SQL } from "../src/telemetry/db"; import { executable, runOps, scratch } from "./harness"; function stubs() { @@ -64,8 +64,16 @@ describe("telemetry", () => { expect(direct.status, direct.stderr).toBe(0); const viaInfisical = runOps(["db", "--"], { env: { ...stub.env, R2_DATA_READ_TOKEN: "" } }); expect(viaInfisical.status, viaInfisical.stderr).toBe(0); + const opening = (environment: string) => [ + "-init", + INIT_SQL, + "-cmd", + expect.stringContaining(`_sketchi-telemetry-${environment}' AS telemetry`), + "-cmd", + `.read "${VIEWS_SQL}"`, + ]; expect(stub.calls()).toEqual([ - { name: "duckdb", args: ["-init", INIT_SQL, "-c", "FROM spans"], env: "preview" }, + { name: "duckdb", args: [...opening("preview"), "-c", "FROM spans"], env: "preview" }, { name: "infisical", args: [ @@ -77,8 +85,7 @@ describe("telemetry", () => { "--silent", "--", "duckdb", - "-init", - INIT_SQL, + ...opening("production"), ], env: "production", },