diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index a1b1580de..02d1e620d 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -77,6 +77,8 @@ import { ProductEventsFunnelResponse, ProductEventsFunnelBreakdownResponse, ProductEventNamesResponse, + ProductEventsForTraceResponse, + ProductEventTraceSamplesResponse, CommitSha, FingerprintHash, ServiceName, @@ -2072,6 +2074,43 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + // Both directions of the trace ↔ product-event link. + .handle("productEventsForTrace", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventsForTrace, tenant, payload) + return new ProductEventsForTraceResponse({ + data: rows.map((row) => ({ + timestamp: String(row.timestamp), + eventName: String(row.eventName), + spanId: String(row.spanId), + serviceName: String(row.serviceName), + userId: String(row.userId), + groupId: String(row.groupId), + visitorId: String(row.visitorId), + sessionId: String(row.sessionId), + // Already decoded as Record by the derived row schema. + attributes: row.attributes, + })), + }) + }), + ) + .handle("productEventTraceSamples", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventTraceSamples, tenant, payload) + return new ProductEventTraceSamplesResponse({ + data: rows.map((row) => ({ + traceId: String(row.traceId), + spanId: String(row.spanId), + timestamp: String(row.timestamp), + serviceName: String(row.serviceName), + userId: String(row.userId), + visitorId: String(row.visitorId), + })), + }) + }), + ) .handle("executeRawSql", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context @@ -2099,7 +2138,9 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query context: "rawSql", }).pipe( // Every statement is audited, however it ended: a refused one as `denied`. - Effect.tap((executed) => audit({ _tag: "rows", rowCount: executed.rowCount })), + Effect.tap((executed) => + audit({ _tag: "rows", rowCount: executed.rowCount }), + ), Effect.tapError((error) => audit( error._tag === "@maple/http/errors/RawSqlValidationError" diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index eaf526c2c..153980dd8 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -182,8 +182,8 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), Object.freeze({ - // TODO(v17): what changed, whether any part is rewritten or any row - // moves, and what this edge does NOT backfill. + // v17: `audit_log` table added (ClickHouse migration 0027). Purely + // additive — nothing is rewritten, no row moves, nothing is backfilled. // // projectRevision is carried forward deliberately — it is a hardcoded // constant that no longer tracks the generator's header, and the identity @@ -194,4 +194,20 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "f19b88567770ee1b67f77d5734de61adbfd3ba907ce8ae28ce65a4da4e544533", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // v18: `product_events` gains `TraceId`/`SpanId` plus a bloom filter, and + // `product_events_traces_mv` projects annotated spans in (ClickHouse + // migration 0028). Metadata-only ALTERs plus a view swap — no part is + // rewritten and no row moves. The trace half IS backfilled from whatever + // `traces` still retains; annotated spans older than that are not. + // + // projectRevision is carried forward deliberately — it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 18, + fingerprint: "09ee43045937c44e", + digest: "09ee43045937c44e89cf65001569497fb2e2d5b3356a8ddc2d81e0a8551bf1b2", + manifestDigest: "2a7d05f4fb19422404264521f06ea9ca2f2106cdce2165899f00433215aca8b0", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index b36f2b4e8..762197d09 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 17 as const +export const LOCAL_SCHEMA_VERSION = 18 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 031deb60a..18c8f4be0 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -53,6 +53,7 @@ import { v13ToV14AiTraceIndexModule } from "./local-store-migrations/v13-to-v14- import { v14ToV15CommitShaVcsRevisionModule } from "./local-store-migrations/v14-to-v15-commit-sha-vcs-revision" import { v15ToV16AiTraceIndexFilterColumnsModule } from "./local-store-migrations/v15-to-v16-ai-trace-index-filter-columns" import { v16ToV17AuditLogModule } from "./local-store-migrations/v16-to-v17-audit-log" +import { v17ToV18ProductEventsFromTracesModule } from "./local-store-migrations/v17-to-v18-product-events-from-traces" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -127,6 +128,7 @@ export const localStoreMigrations: ReadonlyArray = v14ToV15CommitShaVcsRevisionModule, v15ToV16AiTraceIndexFilterColumnsModule, v16ToV17AuditLogModule, + v17ToV18ProductEventsFromTracesModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v17-to-v18-product-events-from-traces.ts b/apps/cli/src/server/local-store-migrations/v17-to-v18-product-events-from-traces.ts new file mode 100644 index 000000000..923a8a5cf --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v17-to-v18-product-events-from-traces.ts @@ -0,0 +1,409 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cloneStoreForStaging } from "./journal-codecs" +import { resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V17, + LOCAL_SCHEMA_V17_MANIFEST, + LOCAL_SCHEMA_V17_SQL, + LOCAL_SCHEMA_V18, + LOCAL_SCHEMA_V18_MANIFEST, + LOCAL_SCHEMA_V18_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +const MODULE_ID = "local-0017-to-0018-product-events-from-traces" as const + +const PRODUCT_EVENTS_TRACE_COLUMNS = [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", +].join(", ") + +// Frozen copy of ClickHouse migration 0028's trace projection, byte-for-byte: +// the backfill and the v18 view must project a span identically. +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * The local mirror of ClickHouse migration 0028: `product_events` gains + * `TraceId`/`SpanId` plus a bloom filter, `product_events_traces_mv` starts + * projecting annotated spans, and the trace half is backfilled from whatever + * `traces` still retains. `product_events_mv` is recreated so its SELECT names + * the two new columns. Every statement is idempotent, so a resume lands in the + * same place. + */ + +interface V17ToV18State { + readonly module: typeof MODULE_ID + readonly version: 1 + readonly rawRows: Readonly> + readonly productEventRows: ProductEventRowCounts + readonly retentionDays?: number +} + +/** Row counts verify re-checks as equalities: `existing` (non-trace rows) must be + * undisturbed, `expectedTrace` (spans matching the backfill filter) must all arrive. */ +interface ProductEventRowCounts { + readonly existing: string + readonly expectedTrace: string +} + +interface V17ToV18Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const isCount = (value: unknown): value is string => typeof value === "string" && /^\d+$/.test(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v17 -> v18 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (!isCount(count)) throw new Error(`v17 -> v18 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v17 -> v18 rawRows contains an unknown table") + return counts +} + +const decodeProductEventRows = (value: unknown): ProductEventRowCounts => { + if (!isRecord(value)) throw new Error("v17 -> v18 productEventRows must be an object") + if (Object.keys(value).some((key) => key !== "existing" && key !== "expectedTrace")) + throw new Error("v17 -> v18 productEventRows contains an unknown field") + if (!isCount(value.existing)) + throw new Error("v17 -> v18 productEventRows.existing must be an unsigned decimal string") + if (!isCount(value.expectedTrace)) + throw new Error("v17 -> v18 productEventRows.expectedTrace must be an unsigned decimal string") + return { existing: value.existing, expectedTrace: value.expectedTrace } +} + +const decodeState = (value: unknown): V17ToV18State => { + if (!isRecord(value)) throw new Error("v17 -> v18 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "productEventRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v17 -> v18 state contains an unknown field") + if (value.module !== MODULE_ID || value.version !== 1) + throw new Error("v17 -> v18 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v17 -> v18 retentionDays must be an integer") + return { + module: MODULE_ID, + version: 1, + rawRows: decodeCounts(value.rawRows), + productEventRows: decodeProductEventRows(value.productEventRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V17ToV18Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v17 -> v18 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const scalarCount = (db: Chdb, sql: string): string => { + const rows = parseJsonEachRow<{ count: string }>(db.query(sql)) + const count = rows[0]?.count + if (!isCount(count)) throw new Error(`v17 -> v18 count query returned no row: ${sql}`) + return count +} + +const productEventRowCounts = (db: Chdb): ProductEventRowCounts => ({ + existing: scalarCount(db, "SELECT toString(count()) AS count FROM product_events"), + expectedTrace: scalarCount( + db, + `SELECT toString(count()) AS count FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ), +}) + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V17_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const { rawRows, productEventRows } = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V17_MANIFEST, retentionDays)) + return { rawRows: rawRowCounts(db), productEventRows: productEventRowCounts(db) } + }, + { schemaSql: LOCAL_SCHEMA_V17_SQL, bootstrapSchema: false }, + ) + return { + module: MODULE_ID, + version: 1, + rawRows, + productEventRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async ( + context: MigrationModuleContext, + state: V17ToV18State, +): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await cloneStoreForStaging(source, target) + } + return state +} + +// Columns, index and both view drops run in the v17 block: the v18 bootstrap is +// all `IF NOT EXISTS`, so it neither adds columns to an existing table nor +// replaces a view that is still there. The backfill runs after the bootstrap +// and writes `product_events` directly, so no view double-fires. +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''") + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''") + db.exec( + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + ) + db.exec("DROP VIEW IF EXISTS product_events_traces_mv") + db.exec("DROP VIEW IF EXISTS product_events_mv") + }, + { schemaSql: LOCAL_SCHEMA_V17_SQL, bootstrapSchema: false }, + ) + return context.openTarget( + (db) => { + // Scoped to the backfill's own source window, matching migration 0028: + // `product_events` keeps 365 days and `traces` 30, so an unbounded delete + // on a re-run would destroy rows the backfill cannot rebuild. The count + // guard keeps an empty `traces` (min() = 1970) from doing the same. + db.exec( + "DELETE FROM product_events WHERE Source = 'trace' AND (SELECT count() FROM traces) > 0 AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + ) + db.exec( + `INSERT INTO product_events (${PRODUCT_EVENTS_TRACE_COLUMNS}) SELECT ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ) + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V18_SQL, bootstrapSchema: true }, + ) +} + +const verify = async ( + context: MigrationModuleContext, + state: V17ToV18State, + _progress: V17ToV18Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V18_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v17 -> v18 raw telemetry verification failed for ${table}`) + } + // Two equalities, not a total: one proves nothing was disturbed, the + // other that every annotated span arrived. + const existing = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source != 'trace'", + ) + if (existing !== state.productEventRows.existing) + throw new Error( + `v17 -> v18 pre-existing product_events row count changed: expected ${state.productEventRows.existing}, found ${existing}`, + ) + const backfilled = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source = 'trace'", + ) + if (backfilled !== state.productEventRows.expectedTrace) + throw new Error( + `v17 -> v18 backfilled trace product_events row count mismatch: expected ${state.productEventRows.expectedTrace}, found ${backfilled}`, + ) + }, + { schemaSql: LOCAL_SCHEMA_V18_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v17-store", + description: "Clone the stopped v17 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-product-event-trace-columns", + description: + "Add TraceId and SpanId to product_events, plus the TraceId bloom filter the trace lookup prunes on", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "backfill-annotated-spans", + description: + "Project every retained span carrying maple.product_event.name into product_events as a Source='trace' row", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "recreate-product-event-views", + description: + "Recreate product_events_mv and create product_events_traces_mv so new rows carry TraceId and annotated spans keep arriving", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v18-schema", + description: + "Verify the v18 physical schema, retained raw telemetry counts, and that the backfill added exactly the annotated spans and disturbed no existing row", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v17 store is cloned byte-for-byte before any DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Read-only source of the backfill; the row count is verified unchanged alongside every other raw telemetry table.", + }, + { + name: "product_events (browser, server and mobile rows)", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Two columns are added as metadata-only defaults, no part is rewritten, and the count of rows whose Source is not 'trace' is verified unchanged after the backfill. Counts, not contents: the byte-level claim rests on ADD COLUMN being metadata-only, which this edge does not independently verify.", + }, + { + // Rebuilt within the raw window, then accrued: `traces` is the source, so + // its shorter retention is the only bound. + name: "product_events (trace rows)", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Every annotated span still inside raw traces retention is re-projected, and the resulting row count is verified to equal the count of matching spans. Annotated spans older than that window are gone from traces and cannot be rebuilt; the table accrues them from the migration forward.", + preservationInterval: "the raw traces retention window", + // Schema default; a custom raw-telemetry floor (`readRawTelemetryRetentionDays`) + // overrides it and the backfill follows the store. + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, +] + +export const v17ToV18ProductEventsFromTracesModule: LocalStoreMigrationModule< + V17ToV18State, + V17ToV18Progress +> = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Add TraceId/SpanId to product_events and project spans annotated with maple.product_event.name into it, backfilled from retained traces", + from: LOCAL_SCHEMA_V17, + to: LOCAL_SCHEMA_V18, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index c1c805916..641900da1 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -16,6 +16,7 @@ import schemaV14Sql from "./schema/local-schema-v14.sql" with { type: "text" } import schemaV15Sql from "./schema/local-schema-v15.sql" with { type: "text" } import schemaV16Sql from "./schema/local-schema-v16.sql" with { type: "text" } import schemaV17Sql from "./schema/local-schema-v17.sql" with { type: "text" } +import schemaV18Sql from "./schema/local-schema-v18.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -77,6 +78,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV15Sql, schemaV16Sql, schemaV17Sql, + schemaV18Sql, ] export interface LocalSchemaSnapshot { @@ -135,6 +137,8 @@ export const LOCAL_SCHEMA_V16_SQL = snapshotAt(16).sql export const LOCAL_SCHEMA_V16_MANIFEST = snapshotAt(16).manifest export const LOCAL_SCHEMA_V17_SQL = snapshotAt(17).sql export const LOCAL_SCHEMA_V17_MANIFEST = snapshotAt(17).manifest +export const LOCAL_SCHEMA_V18_SQL = snapshotAt(18).sql +export const LOCAL_SCHEMA_V18_MANIFEST = snapshotAt(18).manifest export interface LocalSchemaIdentity { readonly version: number @@ -182,6 +186,7 @@ export const LOCAL_SCHEMA_V14 = identityAt(14) export const LOCAL_SCHEMA_V15 = identityAt(15) export const LOCAL_SCHEMA_V16 = identityAt(16) export const LOCAL_SCHEMA_V17 = identityAt(17) +export const LOCAL_SCHEMA_V18 = identityAt(18) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 9d566f057..4713f5271 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced", + "projectRevision": "354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v18.sql b/apps/cli/src/server/schema/local-schema-v18.sql new file mode 100644 index 000000000..a4c01a263 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v18.sql @@ -0,0 +1,2009 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216 +-- localSchemaVersion: 18 + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) AS Model, + coalesce(nullIf(SpanAttributes['gen_ai.agent.name'], ''), SpanAttributes['ai.telemetry.functionId']) AS AgentName, + coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(((StatusCode = 'Error' OR SpanAttributes['error.type'] != '') OR SpanAttributes['gen_ai.response.status'] IN ('failed', 'error'))) AS IsError, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('chat', 'generate_content', 'text_completion', 'fetch_response') OR (((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND NOT ((coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%'))) AND NOT ((lower(SpanName) LIKE '%agent%' OR lower(SpanName) LIKE '%workflow%'))) AND (coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) != '' OR (lower(SpanName) LIKE '%chat%' OR lower(SpanName) LIKE '%completion%'))))) AS IsLlmCall, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('execute_tool') OR (coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND (coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%')))) AS IsToolCall, + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])) AS Tokens, + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cost'], ''), nullIf(SpanAttributes['gen_ai.usage.total_cost'], ''), SpanAttributes['llm.cost.total'])) AS Cost + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 61f1640b7..a4c01a263 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced --- localSchemaVersion: 17 +-- projectRevision: 354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216 +-- localSchemaVersion: 18 CREATE TABLE IF NOT EXISTS ai_trace_index ( OrgId LowCardinality(String), @@ -400,8 +400,11 @@ CREATE TABLE IF NOT EXISTS product_events ( Url String DEFAULT '', ServiceName LowCardinality(String) DEFAULT '', Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, - INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1402,10 +1405,56 @@ SELECT path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom'); +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS SELECT OrgId, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index d1ffa1ba0..f2dbc14cb 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -32,6 +32,7 @@ import { LOCAL_SCHEMA_V15, LOCAL_SCHEMA_V16, LOCAL_SCHEMA_V17, + LOCAL_SCHEMA_V18, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -79,16 +80,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v17 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("b3800f55258f0ae3") - expect(SCHEMA_DIGEST).toBe("b3800f55258f0ae37a52bec6e4fe38be8fa9daebe3c912db2aa6885a4d73fa20") + it("matches the generated v18 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("09ee43045937c44e") + expect(SCHEMA_DIGEST).toBe("09ee43045937c44e89cf65001569497fb2e2d5b3356a8ddc2d81e0a8551bf1b2") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(17) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V17) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(18) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V18) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -174,6 +175,7 @@ describe("current local schema identity", () => { "identity_links_mv", "product_events", "product_events_mv", + "product_events_traces_mv", ]) const errorEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "error_events_mv", @@ -200,7 +202,7 @@ describe("current local schema identity", () => { expect(productEvents?.engine).toBe("MergeTree") expect(productEvents?.orderBy).toBe("(OrgId, Timestamp, VisitorId, SessionId, Seq)") expect(productEvents?.ttl).toContain("365 DAY") - expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id"]) + expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id", "idx_trace_id"]) expect(productEvents?.columns.map((column) => column.name)).toEqual([ "OrgId", "Timestamp", @@ -217,6 +219,10 @@ describe("current local schema identity", () => { "Url", "ServiceName", "Attributes", + // Appended, not inserted: `ALTER TABLE … ADD COLUMN` puts them last, and + // every projection into this table has to match that order. + "TraceId", + "SpanId", ]) const productEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "product_events_mv", @@ -270,6 +276,7 @@ describe("current local schema identity", () => { "ai_trace_index", "ai_trace_index_mv", "audit_log", + "product_events_traces_mv", ]) expect([...v13Names].filter((name) => !currentSchemaNames.has(name))).toEqual([]) const aiTraceIndex = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "ai_trace_index") @@ -318,6 +325,7 @@ describe("local migration registry", () => { "local-0014-to-0015-commit-sha-vcs-revision", "local-0015-to-0016-ai-trace-index-filter-columns", "local-0016-to-0017-audit-log", + "local-0017-to-0018-product-events-from-traces", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -364,7 +372,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 18, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 19, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1370,6 +1378,7 @@ describe("v10 -> v11 product events module", () => { "local-0014-to-0015-commit-sha-vcs-revision", "local-0015-to-0016-ai-trace-index-filter-columns", "local-0016-to-0017-audit-log", + "local-0017-to-0018-product-events-from-traces", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index f41a77989..35452adc6 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 17 and .schema == "b3800f55258f0ae3"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 18 and .schema == "09ee43045937c44e"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index a79cb503b..28aa628e0 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced"; +pub const PROJECT_REVISION: &str = "354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/web/src/api/warehouse/product-events.ts b/apps/web/src/api/warehouse/product-events.ts index 687d2b718..fd70e00e8 100644 --- a/apps/web/src/api/warehouse/product-events.ts +++ b/apps/web/src/api/warehouse/product-events.ts @@ -5,10 +5,13 @@ // `keyBy`, the session step, and the breakdown grouping. import { Effect, Schema } from "effect" +import { TraceId } from "@maple/domain" import { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import { FUNNEL_WIDGET_BREAKDOWN_LIMIT, @@ -63,6 +66,94 @@ const getProductEventNamesEffect = Effect.fn("QueryEngine.getProductEventNames") return { data: result.data satisfies ReadonlyArray } }) +// Trace ↔ product event: an annotated span becomes a `product_events` row carrying +// its `TraceId`, and these two read that column from either end. `TraceId`, not a +// plain string, so a malformed id fails at `decodeInput` rather than the warehouse. +const ProductEventsForTraceInputSchema = Schema.Struct({ + ...TimeWindowFields, + traceId: TraceId, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventsForTraceInput = (typeof ProductEventsForTraceInputSchema)["Encoded"] + +export interface TraceProductEvent { + timestamp: string + eventName: string + /** The annotated span within the trace — deep-links to it in the waterfall. */ + spanId: string + serviceName: string + userId: string + groupId: string + visitorId: string + sessionId: string + /** The span's attributes as projected by `maple.product_event.include` / `prop.*`. */ + attributes: Record +} + +export function getProductEventsForTrace({ data }: { data: GetProductEventsForTraceInput }) { + return getProductEventsForTraceEffect({ data }) +} + +const getProductEventsForTraceEffect = Effect.fn("QueryEngine.getProductEventsForTrace")(function* ({ + data, +}: { + data: GetProductEventsForTraceInput +}) { + const input = yield* decodeInput(ProductEventsForTraceInputSchema, data, "getProductEventsForTrace") + + const result = yield* runWarehouseQuery("productEventsForTrace", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsForTrace({ + payload: new ProductEventsForTraceRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + +const ProductEventTraceSamplesInputSchema = Schema.Struct({ + ...TimeWindowFields, + eventName: Schema.String, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventTraceSamplesInput = (typeof ProductEventTraceSamplesInputSchema)["Encoded"] + +export interface ProductEventTraceSample { + traceId: string + spanId: string + timestamp: string + serviceName: string + userId: string + visitorId: string +} + +export function getProductEventTraceSamples({ data }: { data: GetProductEventTraceSamplesInput }) { + return getProductEventTraceSamplesEffect({ data }) +} + +const getProductEventTraceSamplesEffect = Effect.fn("QueryEngine.getProductEventTraceSamples")(function* ({ + data, +}: { + data: GetProductEventTraceSamplesInput +}) { + const input = yield* decodeInput(ProductEventTraceSamplesInputSchema, data, "getProductEventTraceSamples") + + const result = yield* runWarehouseQuery("productEventTraceSamples", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventTraceSamples({ + payload: new ProductEventTraceSamplesRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + // Dashboard funnel widget (route data source `product_events_funnel`). // // The widget's stored `display.funnel` definition — steps, key, window, an diff --git a/apps/web/src/components/analytics/product-event-trace-samples.tsx b/apps/web/src/components/analytics/product-event-trace-samples.tsx new file mode 100644 index 000000000..eebe6948e --- /dev/null +++ b/apps/web/src/components/analytics/product-event-trace-samples.tsx @@ -0,0 +1,84 @@ +import { Link } from "@tanstack/react-router" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventTraceSamplesResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { formatTimestampInTimezone } from "@/lib/timezone-format" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" +import { ChartBarTrendUpIcon } from "@/components/icons" + +/** + * Recent traces behind one product event. Renders nothing when empty (browser + * and `/v1/events` rows carry no trace, so "none" is not a finding), but a + * failure is shown: the user filtered to this event and asked. + */ +export function ProductEventTraceSamples({ + eventName, + startTime, + endTime, +}: { + eventName: string + startTime: string + endTime: string +}) { + const { effectiveTimezone } = useTimezonePreference() + const result = useAtomValue( + productEventTraceSamplesResultAtom({ data: { eventName, startTime, endTime, limit: 10 } }), + ) + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Traces behind “{eventName}”

+
+
    + {/* Index included: at-least-once ingest can duplicate a row, and + one trace can fire the event from several spans. */} + {response.data.map((sample, index) => ( +
  • + + + {sample.traceId.slice(0, 8)} + + {sample.serviceName === "" ? null : {sample.serviceName}} + {sample.userId || sample.visitorId ? ( + + · {sample.userId || sample.visitorId} + + ) : null} + + {formatTimestampInTimezone(sample.timestamp, { + timeZone: effectiveTimezone, + })} + + +
    • + ))} +
+
+ ) + }) + .onError(() => ( +
+
+ +

Traces behind “{eventName}”

+
+

+ Could not load traces for this event. This is a query failure, not an empty result — + reload to try again. +

+
+ )) + .orElse(() => null) +} diff --git a/apps/web/src/components/traces/trace-product-events.tsx b/apps/web/src/components/traces/trace-product-events.tsx new file mode 100644 index 000000000..6c2a32cde --- /dev/null +++ b/apps/web/src/components/traces/trace-product-events.tsx @@ -0,0 +1,129 @@ +import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventsForTraceResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { ChartBarTrendUpIcon } from "@/components/icons" +import { Badge } from "@maple/ui/components/ui/badge" +import type { TraceProductEvent } from "@/api/warehouse/product-events" + +/** Same margin as `TraceLogsLink`: clock skew can stamp an annotated span just + * outside the root span's window, and the bound keeps the lookup partition-pruned. */ +const WINDOW_MARGIN_MS = 5 * 60 * 1000 + +/** + * The product events this trace produced. Renders nothing while loading, on + * failure, and when there are none (most traces), so it reads as a finding + * rather than another empty section. Clicking an event selects its span. + */ +export function TraceProductEvents({ + traceId, + traceStartTime, + totalDurationMs, + onSelectSpan, +}: { + traceId: string + traceStartTime: string + totalDurationMs: number + onSelectSpan: (spanId: string) => void +}) { + const traceStartMs = parseWarehouseDateTime(traceStartTime) + if (Number.isNaN(traceStartMs)) return null + return ( + + ) +} + +// Split out so an unparseable timestamp never mounts the atom: folding the guard +// in after `useAtomValue` would fail `decodeInput` and export a failure span on +// every render for a query nobody wanted. +function LoadedTraceProductEvents({ + traceId, + startTime, + endTime, + onSelectSpan, +}: { + traceId: string + startTime: string + endTime: string + onSelectSpan: (spanId: string) => void +}) { + const result = useAtomValue(productEventsForTraceResultAtom({ data: { traceId, startTime, endTime } })) + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Product events

+ {response.data.length} +
+
    + {/* Index included: at-least-once ingest can duplicate a row, and the + server-ordered list is never reordered client-side. */} + {response.data.map((event, index) => ( + + ))} +
+
+ ) + }) + .onError(() => null) + .orElse(() => null) +} + +function ProductEventRow({ + event, + onSelectSpan, +}: { + event: TraceProductEvent + onSelectSpan: (spanId: string) => void +}) { + // Same precedence as the funnel person key, so this is the identity the event + // is counted under. + const person = event.userId || event.groupId || event.visitorId + const props = Object.entries(event.attributes) + + const content = ( + <> + {event.eventName} + {event.serviceName === "" ? null : ( + {event.serviceName} + )} + {person === "" ? null : · {person}} + {props.map(([key, value]) => ( + + {key}: {value} + + ))} + + ) + const rowClassName = "flex w-full flex-wrap items-center gap-x-2 gap-y-1 px-3 py-2 text-left text-xs" + + // A row with no span to select is a plain row, not a disabled button: a + // disabled button drops its content out of the tab order. + if (event.spanId === "") { + return
  • {content}
    • + } + + return ( +
  • + +
    • + ) +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index b3023b3cf..de9d9ee2d 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -123,7 +123,11 @@ import { getWebAnalyticsSummary, getWebAnalyticsTimeseries, } from "@/api/warehouse/web-analytics" -import { getProductEventNames } from "@/api/warehouse/product-events" +import { + getProductEventNames, + getProductEventsForTrace, + getProductEventTraceSamples, +} from "@/api/warehouse/product-events" /** * The error union every warehouse server function fails with: the structured @@ -385,6 +389,16 @@ export const productEventNamesResultAtom = makeQueryAtomFamily(getProductEventNa staleTime: 60_000, }) +// A completed trace's product events never change, so this is only ever refetched +// because the trace is still open. 60s matches the route cache behind it. +export const productEventsForTraceResultAtom = makeQueryAtomFamily(getProductEventsForTrace, { + staleTime: 60_000, +}) + +export const productEventTraceSamplesResultAtom = makeQueryAtomFamily(getProductEventTraceSamples, { + staleTime: 60_000, +}) + export const getReplayResultAtom = makeQueryAtomFamily(getReplay, { staleTime: 60_000, }) diff --git a/apps/web/src/routes/analytics/index.tsx b/apps/web/src/routes/analytics/index.tsx index a95d86894..a5d39e4d8 100644 --- a/apps/web/src/routes/analytics/index.tsx +++ b/apps/web/src/routes/analytics/index.tsx @@ -19,6 +19,7 @@ import { type BreakdownDimension, } from "@/components/analytics/analytics-breakdown-panel" import { AnalyticsBotNotice } from "@/components/analytics/analytics-bot-notice" +import { ProductEventTraceSamples } from "@/components/analytics/product-event-trace-samples" import { AnalyticsFilterSidebar } from "@/components/analytics/analytics-filter-sidebar" import { AnalyticsLiveBadge } from "@/components/analytics/analytics-live-badge" import { @@ -588,6 +589,10 @@ function AnalyticsContent({ { id: "events", dimensions: eventDimensions, wide: true }, ] + // Traces behind the filtered event; renders nothing unless the event + // came from an annotated span. + const eventName = filters.eventName + return (
    {cards.map((card) => ( @@ -603,6 +608,15 @@ function AnalyticsContent({ />
    ))} + {eventName === undefined ? null : ( +
    + +
    + )} ) }) diff --git a/apps/web/src/routes/traces/$traceId.tsx b/apps/web/src/routes/traces/$traceId.tsx index 2b2a51319..998605169 100644 --- a/apps/web/src/routes/traces/$traceId.tsx +++ b/apps/web/src/routes/traces/$traceId.tsx @@ -13,6 +13,7 @@ import { QueryErrorState } from "@/components/common/query-error-state" import { TraceViewTabs } from "@maple/ui/components/traces/trace-view-tabs" import { SpanDetailPanel } from "@/components/traces/span-detail-panel" import { TraceAnatomyStrip } from "@/components/traces/trace-anatomy-strip" +import { TraceProductEvents } from "@/components/traces/trace-product-events" import { Skeleton } from "@maple/ui/components/ui/skeleton" import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@maple/ui/components/ui/resizable" import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@maple/ui/components/ui/sheet" @@ -245,6 +246,18 @@ function TraceDetailContent({ [search.spanId, navigate], ) + // The product-events panel knows a span id, not a `SpanNode`. + const handleSelectSpanId = React.useCallback( + (spanId: string) => { + if (search.spanId === spanId) return + navigate({ + search: (prev: Record) => ({ ...prev, spanId }), + replace: true, + }) + }, + [search.spanId, navigate], + ) + const handleCloseSpanDetails = React.useCallback(() => { navigate({ search: (prev: Record) => ({ ...prev, spanId: undefined }), @@ -335,6 +348,13 @@ function TraceDetailContent({ commitSha={commitSha} /> + + {isMobile ? ( // A 60/40 side-by-side split leaves each pane ~150px on a phone. Give the waterfall // the full width and float the span detail over it instead. diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index 60198e2ca..a8c364d0c 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -277,3 +277,171 @@ the `product_events_funnel` route params (`ProductEventsFunnelWidgetParams` in ` which the browser server function and the share API's route plan both decode. With a breakdown the route answers `{ name, value, group }` rows (top 6 groups by step-1 count) and the funnel chart draws one bar per group per step with a legend. + +## Product events from traces — annotate in code (2026-09) + +The fourth feed into `product_events`, after browser (`session_events` MV), server and mobile +(`POST /v1/events`). A team marks a span they already emit and it becomes a funnel step that +links back to the request that performed it. + +```ts +span.setAttributes({ + "maple.product_event.name": "checkout_completed", // required — presence is the predicate + "maple.product_event.user_id": user.id, // optional identity + "maple.product_event.group_id": org.id, + "maple.product_event.visitor_id": anonId, + "maple.product_event.url": req.url, // optional page context +}) +``` + +**Every other attribute on the span becomes an event property by default.** Nothing has to be +declared to get started — whatever the team already sets on the span (`plan`, `order.total`, the +full HTTP/DB semconv surface) lands in `Attributes` and is available to funnel breakdowns. The +`maple.product_event.*` control keys are stripped, since they are already promoted to their own +columns. + +Two optional controls narrow or replace that default. Both are themselves span attributes: a +materialized view is static SQL per cluster and has no per-org config to read. + +```ts +"maple.product_event.include": "plan,seats" // ONLY these span keys (whitespace trimmed) +"maple.product_event.prop.plan": "pro" // explicit prop, merged over the base, wins ties +"maple.product_event.include": "" // and together: full overwrite +``` + +Three tiers out of one mechanism rather than three modes to pick between: + +| `include` | `prop.*` | `Attributes` | +| --- | --- | --- | +| absent | — | every span attribute | +| absent | set | every span attribute, with the props overriding on a key collision | +| `"plan,seats"` | — | only `plan` and `seats` | +| `""` | set | only the props — the overwrite case | + +`include` switches on **key presence**, not on a non-empty value, which is what makes the empty +string mean "no span attributes" rather than "no filter". There is no separate replace flag to get +wrong, and `mapUpdate(base, props)` argument order is the override rule — swapped, an override +would be discarded exactly when the key it meant to correct was already present. + +Verified against ClickHouse 26.2, all three tiers: + +| Scenario | Span attributes | Result | +| --- | --- | --- | +| default | `http.method`, `plan=free`, `seats=5`, `prop.plan=pro` | `{http.method, seats, plan:'pro'}` | +| `include: "plan, seats"` | + `noise` | `{plan:'free', seats:'5'}` | +| `include: ""` + `prop.plan=pro` | `http.method`, `plan=free` | `{plan:'pro'}` | + +The contract lives in one place — `packages/domain/src/tinybird/product-event-attributes.ts` — +and is read by exactly two consumers that must agree byte for byte: `productEventsTracesMv` +(managed orgs, via `tinybird deploy`) and the frozen copy inside ClickHouse migration 0028 (BYO +clusters). The migration's copy is deliberately NOT imported from the constant: a delta migration +describes one step in history, and a shared constant would silently rewrite what 0028 did the next +time the live projection changes. + +### Why an attribute and not a UI action + +A product event has to be emitted by the code path that performed the thing, at the moment it +performed it. Marking a trace by hand in the UI marks *one sampled trace*, cannot be replayed over +history, and puts a mutable user-authored row into an append-only fact table. An attribute marks +every trace the path produces, applies retroactively across the whole `traces` retention window, +and is reviewable in the customer's own diff. There is no second store and no write path from the +dashboard — the span is the record, the product event is its projection. + +### The link + +`product_events` gained `TraceId`/`SpanId` (migration 0028, `DEFAULT ''`, appended, plus a +`bloom_filter` on `TraceId`). Non-empty only on `Source = 'trace'` rows. Real columns rather than +`Attributes` keys because both directions filter on them, and a `Map` lookup on this table reads +the whole map per row — the exact cost `product_events` was split out of `session_events` to avoid. + +| Direction | Query | Surface | +| --- | --- | --- | +| trace → its product events | `productEventsForTraceQuery` | trace detail page, under the anatomy strip | +| event → the traces behind it | `productEventTraceSamplesQuery` | `/analytics`, when the `eventName` filter is set | + +Both are `profile: "list"` with a flat `cache: 60` rather than `timeRangeCache`: they are point +lookups whose answer does not widen with the range asked about, and a completed trace's events +never change at all. + +### What it costs + +The MV predicate is one `Map` value read per incoming span, on the same block every other `traces` +MV already fires on. An MV sees the insert block, not the table, so `idx_span_attr_keys` does not +help it — this is a real, deliberately small per-span ingest cost. + +Copying the whole `SpanAttributes` map **by default** is the deliberate expensive choice. A server +span's map is dominated by HTTP/DB semconv keys, and `product_events` keeps 365 days against raw +`traces`' 30 — so an annotated span's attributes outlive the span itself by a factor of twelve. What +the default buys is that nothing has to be declared to get a useful event; `include` is the lever +for a team that has measured the cost and wants it back, and it is a one-line change on the span +rather than a schema migration. + +The practical consequence to watch: attribute pickers over product events list the span's whole +semconv surface for any team that has not set `include`. If that becomes the dominant cost across +orgs rather than for one of them, the lever is a per-org key denylist at the MV — the per-span +`include` handles the single-team case already. + +The whole `Attributes` expression only evaluates for rows passing the `WHERE`, i.e. annotated spans, +so its cost is paid per product event rather than per span. The predicate itself stays one map +lookup. + +### Rollout + +1. **Managed**: `bun run --cwd apps/api tinybird:deploy` creates `product_events_traces_mv` and + adds the two columns, then an explicit `tb` populate from `traces` (bounded by its 30-day TTL). + Blocked on the same manual step the rest of this document's checklist is — see + `project_product_events_tinybird_rollout_pending`. + + **The `FORWARD_QUERY` on `product_events` is what makes this deploy safe, and it is not + optional.** Adding two defaulted columns is *not* a free change here: without the forward query + Tinybird satisfies the new schema by rebuilding the table from the datasources that feed it, and + both (`session_events`, `traces`) keep 30 days against `product_events`' 365. It says so and + proceeds anyway — + + > it is going to be backfilled using the following datasources which would lead to a deleting + > historical data + + — which on this dual-fed table is worse than it sounds: the server and mobile rows arrive by + `POST /v1/events` and have **no source datasource at all**, so a rebuild drops them at every age, + not just past 30 days. Verified against a real deploy, not inferred. + + Two traps around it. `DEPLOYMENT_METHOD alter` on `product_events_mv` does **not** substitute — + tested, and the same data-loss warning returns, because it is the datasource schema change that + triggers the source backfill, not the view's. And once the forward query is in place Tinybird + suggests the inverse ("could be applied with ALTER TABLE and no data movement at promotion time + if you remove the FORWARD_QUERY"); following that suggestion reintroduces the loss. Per the + Tinybird rules the forward query can be deleted in a *later* deploy, once this one has compacted. + + **The populate is one-shot and overlap-prone.** Unlike BYO and local, the managed surface has no + `DELETE WHERE Source = 'trace'` step, so running it twice double-inserts, and running it after + the MV is already live double-counts every annotated span ingested between MV creation and the + populate's own snapshot. BYO risks a gap; managed risks duplicates. Same caveat 0014 and 0021 + accepted — but on a table feeding customer-visible funnels, a double-counted conversion is worse + than a missing one. Populate once, immediately after deploy, and if it fails partway prefer + deleting the trace rows by hand over re-running it blind. +2. **BYO ClickHouse**: migration 0028, `requiredForIngest: false`. That is safe for one reason + worth knowing before anyone touches `datasources.ts`: `TraceId`/`SpanId` are declared with **no + `jsonPath`**, so the insert-mapping generator omits them and the Rust gateway's + `INSERT INTO product_events (…)` never names them — a cluster stamped below 28 still accepts + every row it sends. Give those columns a `jsonPath` and the flag becomes a data-loss bug: the + readiness gate still says 21, so unmigrated BYO orgs keep routing to their own cluster, where + the INSERT fails on the unknown column, retries, trips the breaker and drops the batch. + Backfills the trace half from `traces` itself. +3. **Local CLI**: local schema v17 → v18, same backfill. + +Both BYO and local scope their idempotency `DELETE` to `Timestamp >= (SELECT min(Timestamp) FROM +traces)` rather than deleting all trace rows. `product_events` keeps 365 days and `traces` 30, so an +unbounded delete on a *late* re-apply would clear a year of funnel history and rebuild only a month +of it. The delete is also guarded by `(SELECT count() FROM traces) > 0`: `min()` over an empty table +is 1970, which would turn the bound back into "everything". + +### Not in this cut + +- **No MCP tool.** `list_product_events` still returns names only, and `inspect_trace` does not + surface a trace's product events. An agent cannot walk the link yet; the queries and the HTTP + routes it would sit on both exist. +- **No SDK helper.** Teams set the attributes by hand on whatever span API they already use. A + `markProductEvent(span, name, { props, include })` in `@maple-dev/effect-sdk` would be the obvious + next step — it is a wrapper over `setAttributes` that builds the `prop.*` keys and joins + `include`, not new machinery, and it is where the empty-string overwrite idiom would get a name + (`attributes: "none"`) instead of being a documented convention. diff --git a/packages/domain/src/clickhouse/migrations/0028_product_events_from_traces.ts b/packages/domain/src/clickhouse/migrations/0028_product_events_from_traces.ts new file mode 100644 index 000000000..f1dbf7595 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0028_product_events_from_traces.ts @@ -0,0 +1,153 @@ +import type { BackfillSpec } from "../backfill" + +// Frozen copy of the trace→product-event projection as of this migration (the +// live one is `tinybird/product-event-attributes.ts`). Deliberately not imported: +// a delta migration describes one step in history. Shared within this file so +// the backfill and the view project a span identically. +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * Backfill of the trace half. Row-wise, so any chunk boundary is safe, and + * bounded by `traces`' 30-day TTL against `product_events`' 365: an org sees + * ~30 days of history immediately and accrues the rest going forward. + */ +export const productEventsTracesBackfill: BackfillSpec = { + kind: "backfill", + target: "product_events", + columns: [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", + ], + from: "traces", + tsColumn: "Timestamp", + select: PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + where: PRODUCT_EVENTS_TRACE_FILTER, +} + +/** + * Migration 0028 — product events annotated in code. A span carrying + * `maple.product_event.name` becomes a `product_events` row with its `TraceId`, + * so it steps in a funnel and links back to the trace that produced it. + * + * 1. `product_events` gains `TraceId`/`SpanId` (`DEFAULT ''`, appended) plus a + * bloom filter on `TraceId`. + * 2. `product_events_traces_mv` projects annotated spans in; the trace half is + * backfilled from `traces`' 30-day window. + * + * `product_events_mv` is recreated so its SELECT names all 17 columns. + * Re-runnable: `IF NOT EXISTS` throughout and a `DELETE` scoped to the + * backfill's own window. + * + * **BYO ClickHouse only.** Managed orgs get the view via `tinybird deploy`, with + * the populate as an explicit `tb` step (see 0014 and 0021). + * + * `requiredForIngest: false` rests on `TraceId`/`SpanId` having NO `jsonPath` in + * `datasources.ts`: the insert-mapping generator omits them, so the gateway's + * `INSERT INTO product_events (…)` never names them and a cluster stamped below + * 28 still accepts every row. Give them a path and unmigrated BYO orgs drop every + * `/v1/events` batch on the unknown column. + */ +export const migration_0028_product_events_from_traces = { + version: 28, + description: + "Add TraceId/SpanId to product_events and materialize product events from spans carrying the maple.product_event.name attribute", + requiredForIngest: false, + statements: [ + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''", + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''", + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + // Dropped and IMMEDIATELY recreated: every `session_events` row ingested + // while the browser view is gone is never projected, so the gap must not + // span the chunked backfill below. + "DROP VIEW IF EXISTS product_events_mv", + // Frozen copy of the browser projection as of 0021, plus the two new columns. + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId +FROM session_events +WHERE Type IN ('navigation', 'custom')`, + "DROP VIEW IF EXISTS product_events_traces_mv", + // Idempotency for the trace half only, bounded by the backfill's own window: + // `traces` keeps 30 days and `product_events` 365, so an unbounded delete on + // a late re-run would destroy rows the backfill cannot rebuild. The count + // guard keeps an empty `traces` (min() = 1970) from doing the same. + "DELETE FROM product_events WHERE Source = 'trace' AND (SELECT count() FROM traces) > 0 AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + productEventsTracesBackfill, + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT +${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} +FROM traces +WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 311ac48f2..ed3251b80 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -33,6 +33,7 @@ import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_index_filter_columns" import { migration_0027_audit_log } from "./0027_audit_log" +import { migration_0028_product_events_from_traces } from "./0028_product_events_from_traces" import { migration_0021_product_events } from "./0021_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" @@ -50,10 +51,10 @@ describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, + 27, 28, ]) - expect(migrations.at(-1)).toBe(migration_0027_audit_log) - expect(latestMigrationVersion).toBe(27) + expect(migrations.at(-1)).toBe(migration_0028_product_events_from_traces) + expect(latestMigrationVersion).toBe(28) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -80,6 +81,7 @@ describe("ClickHouse migrations", () => { // 0026 widens the same MV-populated ai_trace_index and rebuilds its view. expect(migration_0026_ai_trace_index_filter_columns.requiredForIngest).toBe(false) expect(migration_0027_audit_log.requiredForIngest).toBe(false) + expect(migration_0028_product_events_from_traces.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index bbf6eb295..9257f9233 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -26,6 +26,7 @@ import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_index_filter_columns" import { migration_0027_audit_log } from "./0027_audit_log" +import { migration_0028_product_events_from_traces } from "./0028_product_events_from_traces" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -84,6 +85,7 @@ export const migrations: ReadonlyArray = [ migration_0025_commit_sha_vcs_revision, migration_0026_ai_trace_index_filter_columns, migration_0027_audit_log, + migration_0028_product_events_from_traces, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 05306fac3..29a134bdf 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced" as const +export const projectRevision = "354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_trace_index (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SessionId String,\n VendorId LowCardinality(String),\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Model LowCardinality(String),\n AgentName LowCardinality(String),\n ToolName LowCardinality(String),\n SpanId String,\n ParentSpanId String,\n Duration UInt64,\n IsError UInt8,\n IsLlmCall UInt8,\n IsToolCall UInt8,\n Tokens Float64,\n Cost Float64\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -20,7 +20,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS metrics_gauge (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n BucketCounts Array(UInt64),\n ExplicitBounds Array(Float64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_sum (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n AggregationTemporality Int32,\n IsMonotonic Bool\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n TraceId String DEFAULT '',\n SpanId String DEFAULT '',\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4,\n INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_external_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_map_children (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, ParentSpanId, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", @@ -58,7 +58,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 1899d65ea..7933ac4cd 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced" as const +export const projectRevision = "354a3f51b4fc9cef85b49c7624d6c863e35c7786dbcf92552f593ca9493d8216" as const export const datasources = [ { @@ -87,7 +87,7 @@ export const datasources = [ { name: "product_events", content: - "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map()\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4", + "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map(),\n TraceId String `json:$.TraceId` DEFAULT '',\n SpanId String `json:$.SpanId` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, Timestamp, Source, SessionId, Seq, VisitorId, UserId, GroupId, Kind, EventName,\n \t\tHost, PagePath, Url, ServiceName, Attributes,\n \t\tdefaultValueOfTypeName('String') AS TraceId,\n \t\tdefaultValueOfTypeName('String') AS SpanId", }, { name: "service_address_resolutions_hourly", @@ -280,7 +280,12 @@ export const pipes = [ { name: "product_events_mv", content: - "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + }, + { + name: "product_events_traces_mv", + content: + "DESCRIPTION >\n Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, attributes (narrowed by maple.product_event.include, merged with maple.product_event.prop.*), service and TraceId/SpanId so the event links back to the trace that produced it.\n\nNODE product_events_traces_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''\n\nTYPE MATERIALIZED\nDATASOURCE product_events", }, { name: "service_external_edges_hourly_mv", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index e6368b324..7371fda1b 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -43,6 +43,22 @@ const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0 }), ) +/** + * A `LIMIT` a client may ask for. The builder INLINES it into the SQL text, so + * `-1` or `1e21` would be a syntax error (a 500) and `1e9` an unbounded scan; + * the ceiling lives here because the internal API is reachable by any client. + */ +const RowLimit = Schema.Number.check( + Schema.isInt(), + Schema.isGreaterThan(0), + Schema.isLessThanOrEqualTo(1000), +).pipe( + Schema.annotate({ + identifier: "RowLimit", + description: "Maximum rows to return: a whole number between 1 and 1000.", + }), +) + // Dedicated endpoint schemas /** Shared primitives for filtered list/facet endpoints. */ @@ -1753,6 +1769,66 @@ export class ProductEventNamesResponse extends Schema.Class( + "ProductEventsForTraceRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + traceId: TraceId, + /** Default 50, max 1000. */ + limit: Schema.optional(RowLimit), +}) {} + +export class ProductEventsForTraceResponse extends Schema.Class( + "ProductEventsForTraceResponse", +)({ + data: Schema.Array( + Schema.Struct({ + timestamp: Schema.String, + eventName: Schema.String, + /** The annotated span within the trace. */ + spanId: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + groupId: Schema.String, + visitorId: Schema.String, + sessionId: Schema.String, + /** The span's attributes as projected by `maple.product_event.include` / `prop.*`. */ + attributes: Schema.Record(Schema.String, Schema.String), + }), + ), +}) {} + +/** Recent traces behind one event name — the analytics side of the same link. */ +export class ProductEventTraceSamplesRequest extends Schema.Class( + "ProductEventTraceSamplesRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + eventName: Schema.String, + /** Default 20, max 1000. */ + limit: Schema.optional(RowLimit), +}) {} + +export class ProductEventTraceSamplesResponse extends Schema.Class( + "ProductEventTraceSamplesResponse", +)({ + data: Schema.Array( + Schema.Struct({ + traceId: Schema.String, + spanId: Schema.String, + timestamp: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + visitorId: Schema.String, + }), + ), +}) {} + export class PodFacetsRequest extends Schema.Class("PodFacetsRequest")({ startTime: TinybirdDateTime, endTime: TinybirdDateTime, @@ -2710,6 +2786,20 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("productEventsForTrace", "/product-events-for-trace", { + payload: ProductEventsForTraceRequest, + success: ProductEventsForTraceResponse, + error: queryEngineEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("productEventTraceSamples", "/product-event-trace-samples", { + payload: ProductEventTraceSamplesRequest, + success: ProductEventTraceSamplesResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("executeRawSql", "/execute-raw-sql", { payload: RawSqlExecuteRequest, diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 825ac1af5..a3ff3504c 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2331,7 +2331,33 @@ export const productEvents = defineDatasource("product_events", { Attributes: column(t.map(t.string(), t.string()).defaultExpr("map()"), { jsonPath: "$.attributes", }), + /** + * The trace this event was derived from — set on `Source = 'trace'` rows, + * `''` otherwise. A real column because both link directions filter on it + * and a `Map` lookup reads the whole map per row. Last because + * `ALTER TABLE … ADD COLUMN` appends. + * + * NO `jsonPath`, deliberately: only `product_events_traces_mv` and its + * backfill write these. The insert-mapping generator skips path-less + * columns, so the gateway's INSERT never names them and migration 0028 can + * stay `requiredForIngest: false`. Give them a path and every `/v1/events` + * batch for a BYO cluster stamped below 28 is rejected. + */ + TraceId: t.string().default(""), + /** The annotated span within {@link TraceId}. `''` on non-trace rows. */ + SpanId: t.string().default(""), }, + // REQUIRED, proven against a real deploy: without it Tinybird REBUILDS this + // table from its 30-day sources to satisfy the new columns, dropping history + // past 30 days and every `/v1/events` row at any age (they have no source). + // `DEPLOYMENT_METHOD alter` on the view does not substitute — tested. Do not + // follow Tinybird's later suggestion to drop it in favour of ALTER TABLE. + // Every column must be listed; the two new ones take their type default. + forwardQuery: `SELECT + OrgId, Timestamp, Source, SessionId, Seq, VisitorId, UserId, GroupId, Kind, EventName, + Host, PagePath, Url, ServiceName, Attributes, + defaultValueOfTypeName('String') AS TraceId, + defaultValueOfTypeName('String') AS SpanId`, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", sortingKey: ["OrgId", "Timestamp", "VisitorId", "SessionId", "Seq"], @@ -2356,6 +2382,14 @@ export const productEvents = defineDatasource("product_events", { type: "bloom_filter", granularity: 4, }, + { + // The trace view looks up by id alone; near-unique values and `''` on + // most rows make a bloom filter prune hard and stay cheap. + name: "idx_trace_id", + expr: "TraceId", + type: "bloom_filter", + granularity: 4, + }, ], }) diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 61d93ddc8..c20a40b1e 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -47,6 +47,7 @@ import { DB_SYSTEM_ATTR_SQL, } from "./db-query-shape-sql" import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_VENDOR_ID_ATTR } from "../gen-ai" +import { PRODUCT_EVENTS_TRACE_FILTER, PRODUCT_EVENTS_TRACE_PROJECTION_SQL } from "./product-event-attributes" import { DEPLOYMENT_ENV_SQL, MESSAGING_DESTINATION_SQL } from "./semconv-renames" import { GENAI_AGENT_NAME_SQL, @@ -1653,7 +1654,9 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom') `, @@ -1661,6 +1664,30 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { ], }) +/** + * Populates `product_events` from spans carrying `maple.product_event.name` — + * the only feed that carries `TraceId`. The predicate is one map lookup per + * incoming span (an MV sees the insert block, so no skip index helps). Column + * order must match the `product_events` SCHEMA order, enforced by + * `materialized-projection-order.test.ts`. + */ +export const productEventsTracesMv = defineMaterializedView("product_events_traces_mv", { + description: + "Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, attributes (narrowed by maple.product_event.include, merged with maple.product_event.prop.*), service and TraceId/SpanId so the event links back to the trace that produced it.", + datasource: productEvents, + nodes: [ + node({ + name: "product_events_traces_mv_node", + sql: ` + SELECT + ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} + FROM traces + WHERE ${PRODUCT_EVENTS_TRACE_FILTER} + `, + }), + ], +}) + /** * Populates `identity_links` from `session_replays` rows that carry both a * visitor and a user id. diff --git a/packages/domain/src/tinybird/product-event-attributes.test.ts b/packages/domain/src/tinybird/product-event-attributes.test.ts new file mode 100644 index 000000000..13b548253 --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest" +import { + PRODUCT_EVENTS_TRACE_FILTER, + PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + PRODUCT_EVENT_ATTRIBUTE_NAMESPACE, + PRODUCT_EVENT_INCLUDE_KEY, + PRODUCT_EVENT_NAME_KEY, + PRODUCT_EVENT_PROP_PREFIX, + PRODUCT_EVENT_SOURCE_TRACE, +} from "./product-event-attributes" +import { productEvents } from "./datasources" + +describe("product event span attributes", () => { + it("namespaces every key under maple.", () => { + // The `maple.*` vendor namespace is the convention every custom attribute + // in the repo follows, and it is what keeps these from colliding with an + // OTel semconv key that might later mean something else. + expect(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE.startsWith("maple.")).toBe(true) + expect(PRODUCT_EVENT_NAME_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_INCLUDE_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_PROP_PREFIX.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + }) + + it("casts the base map to the target key type", () => { + // Not cosmetic: `SpanAttributes` is keyed `LowCardinality(String)` and + // `product_events.Attributes` is keyed plain `String`. Without the CAST the + // MV's SELECT has a different type from the column it writes, and + // `mapUpdate` has two differently-keyed maps to merge. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("'Map(String, String)'") + }) + + it("switches the allow-list on key PRESENCE, not on a non-empty value", () => { + // `include: ''` must mean "no span attributes"; a `!= ''` would silently + // turn the documented overwrite into copy-everything. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}')`, + ) + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).not.toContain( + `SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'] != ''`, + ) + }) + + it("trims the allow-list entries", () => { + // `"plan, seats"` is what a human writes. Without the trim the second key + // never matches and the prop silently vanishes from every event. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("trimBoth") + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'])`, + ) + }) + + it("strips exactly the prop prefix and nothing more", () => { + // ClickHouse `substring` is 1-indexed, so the offset is length + 1; off by + // one either way silently yields a valid map with wrong keys. + const offset = /substring\(k, (\d+)\)/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL)?.[1] + expect(offset).toBe(String(PRODUCT_EVENT_PROP_PREFIX.length + 1)) + }) + + it("merges props over the base so an explicit prop wins a collision", () => { + // Argument order in `mapUpdate(base, props)` IS the override rule. Swapped, + // a team overriding a derived value would find their override discarded + // exactly when the key they wanted to correct was already present. + const merge = /mapUpdate\(\s*CAST\(/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL) + expect(merge, "props must be the SECOND mapUpdate argument").not.toBeNull() + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapApply")).toBeGreaterThan( + PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapUpdate"), + ) + }) + + it("keeps the control namespace out of the copied attributes", () => { + // Once `prop.*` exists, leaving the namespace in means every explicit prop + // appears twice — as `plan` from the merge and as + // `maple.product_event.prop.plan` from the base — and name/user_id + // duplicate columns this same SELECT already promotes. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}')`, + ) + }) + + it("filters on a non-empty name rather than key presence", () => { + // `mapContains` would admit a span whose attribute is set to '' and mint a + // nameless event — a row no funnel can step on and no reader can attribute. + expect(PRODUCT_EVENTS_TRACE_FILTER).toBe(`SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''`) + }) + + it("projects the product_events columns in schema order", () => { + // The MV body is checked structurally by materialized-projection-order, + // which reads the generated manifest. This asserts the same thing about the + // shared constant itself, so a bad edit fails before the manifest is + // regenerated rather than after. + // + // Split on TOP-LEVEL commas only. A line-wise scan was enough while every + // projected column was one line; `Attributes` is now a nested expression + // whose inner `mapFilter`/`arrayMap` lambdas contain both commas and bare + // identifiers, and a naive scan reads `k` and `SpanAttributes` as columns. + const topLevelParts = (input: string): ReadonlyArray => { + const parts: string[] = [] + let start = 0 + let depth = 0 + let inString = false + for (let index = 0; index < input.length; index++) { + const char = input[index] + if (char === "'" && input[index - 1] !== "\\") { + inString = !inString + continue + } + if (inString) continue + if (char === "(") depth++ + if (char === ")") depth-- + if (char === "," && depth === 0) { + parts.push(input.slice(start, index)) + start = index + 1 + } + } + parts.push(input.slice(start)) + return parts.map((part) => part.trim()).filter((part) => part.length > 0) + } + + const projected = topLevelParts(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).map((part) => { + const aliased = /\bAS ([A-Za-z][A-Za-z0-9_]*)$/.exec(part) + return aliased ? aliased[1]! : part + }) + expect(projected).toEqual(Object.keys(productEvents._schema)) + }) + + it("carries a Source distinct from the other three feeds", () => { + // `Source` is the provenance column every reader branches on to tell an + // annotated span from a browser page view or a POST /v1/events row. + expect(PRODUCT_EVENT_SOURCE_TRACE).toBe("trace") + expect(["browser", "server", "mobile"]).not.toContain(PRODUCT_EVENT_SOURCE_TRACE) + }) +}) diff --git a/packages/domain/src/tinybird/product-event-attributes.ts b/packages/domain/src/tinybird/product-event-attributes.ts new file mode 100644 index 000000000..55ce35408 --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.ts @@ -0,0 +1,130 @@ +/** + * The span-attribute contract that turns an instrumented span into a product + * event. A customer marks a span they already emit: + * + * ```ts + * span.setAttributes({ + * "maple.product_event.name": "checkout_completed", + * "maple.product_event.user_id": user.id, + * "maple.product_event.include": "plan,seats", // optional: ONLY these span keys + * "maple.product_event.prop.plan": "pro", // optional: explicit prop, wins ties + * }) + * ``` + * + * `product_events_traces_mv` projects it into `product_events` with + * `Source = 'trace'` and its `TraceId`/`SpanId`. The span's other attributes + * become the event's properties by default; `include` narrows that base, + * `prop.*` merges over it, and an EMPTY `include` leaves only the props. + * Full write-up in `docs/product-events-funnels.md`. + * + * Read in two places that must agree byte for byte: `productEventsTracesMv` + * (managed) and the frozen copy in ClickHouse migration 0028 (BYO). + */ + +/** Vendor namespace prefix. Every key below starts with it. */ +export const PRODUCT_EVENT_ATTRIBUTE_NAMESPACE = "maple.product_event." + +/** + * Required. Its presence — a non-empty value — is the whole predicate: a span + * carrying it becomes a product event, a span without it is ignored. The value + * becomes `EventName`, i.e. the funnel step key. + */ +export const PRODUCT_EVENT_NAME_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}name` + +/** Optional identity. Absent keys project to `''`, which means "unidentified". */ +export const PRODUCT_EVENT_USER_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}user_id` +export const PRODUCT_EVENT_GROUP_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}group_id` +export const PRODUCT_EVENT_VISITOR_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}visitor_id` + +/** + * Optional page context, for events that belong to a URL (a server-rendered + * checkout, a webhook that knows the page it came from). + */ +export const PRODUCT_EVENT_URL_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}url` + +/** + * Optional allow-list: comma-separated span attribute keys (whitespace trimmed), + * and only those are copied into `Attributes`. PRESENCE switches it, not the + * value, so present-and-empty means "no span attributes at all" — the + * documented way to overwrite with {@link PRODUCT_EVENT_PROP_PREFIX} props. + */ +export const PRODUCT_EVENT_INCLUDE_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}include` + +/** + * Prefix for explicit props: `maple.product_event.prop.plan` lands in + * `Attributes` as `plan`, merged over the base map and winning a collision. + */ +export const PRODUCT_EVENT_PROP_PREFIX = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}prop.` + +/** + * OTel's own session key, read as a fallback so a browser-originated trace can + * stitch to the same session its `session_events` rows carry. + */ +export const PRODUCT_EVENT_SESSION_ID_KEY = "session.id" + +/** + * `Source` value for a trace-derived row. Joins `browser` (session_events MV), + * `server` and `mobile` (`POST /v1/events`). + */ +export const PRODUCT_EVENT_SOURCE_TRACE = "trace" + +/** + * The `product_events` projection of one annotated span, in SCHEMA column order + * (`TraceId`/`SpanId` last, where `ADD COLUMN` put them). `Kind = 'custom'` + * because a trace-derived event is a tracked event, not a page view; provenance + * lives in `Source`. + * + * `Attributes` = `mapUpdate(base, props)`, so props win a collision: + * base = span attributes minus the `maple.product_event.*` namespace, + * narrowed to `include`'s list when that key is PRESENT (`has(mapKeys)`) + * props = `maple.product_event.prop.*`, prefix stripped + * + * Copying the whole map by default is the deliberate expensive choice; `include` + * is the lever. The expression only runs for rows passing the WHERE. + */ +export const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + '${PRODUCT_EVENT_SOURCE_TRACE}' AS Source, + SpanAttributes['${PRODUCT_EVENT_SESSION_ID_KEY}'] AS SessionId, + 0 AS Seq, + SpanAttributes['${PRODUCT_EVENT_VISITOR_ID_KEY}'] AS VisitorId, + SpanAttributes['${PRODUCT_EVENT_USER_ID_KEY}'] AS UserId, + SpanAttributes['${PRODUCT_EVENT_GROUP_ID_KEY}'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] AS EventName, + domain(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS Host, + path(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS PagePath, + SpanAttributes['${PRODUCT_EVENT_URL_KEY}'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}') + AND ( + NOT has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, ${PRODUCT_EVENT_PROP_PREFIX.length + 1}), v), + mapFilter((k, v) -> startsWith(k, '${PRODUCT_EVENT_PROP_PREFIX}'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +/** + * The predicate, also shared. `!= ''` rather than `mapContains`: a key present + * with an empty value would otherwise mint an event with no name, which is a row + * no funnel can step on and no reader can attribute. + */ +export const PRODUCT_EVENTS_TRACE_FILTER = `SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''` diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 9f925c0d7..6dd4467df 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1176,6 +1176,26 @@ SELECT LIMIT 100 FORMAT JSON +-- builder:product-events:productEventsForTraceQuery:default [d151e174] +SELECT + Timestamp AS timestamp, + EventName AS eventName, + SpanId AS spanId, + ServiceName AS serviceName, + UserId AS userId, + GroupId AS groupId, + VisitorId AS visitorId, + SessionId AS sessionId, + Attributes AS attributes + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND TraceId = '4bf92f3577b34da6a3ce929d0e0e4736' + ORDER BY timestamp ASC, spanId ASC + LIMIT 50 + FORMAT JSON + -- builder:product-events:productEventsFunnelBreakdownQuery:attribute-session-step [ac39fa69] SELECT group AS group, @@ -1540,6 +1560,24 @@ SELECT ORDER BY step ASC FORMAT JSON +-- builder:product-events:productEventTraceSamplesQuery:default [ee1608d5] +SELECT + TraceId AS traceId, + SpanId AS spanId, + Timestamp AS timestamp, + ServiceName AS serviceName, + UserId AS userId, + VisitorId AS visitorId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND EventName = 'checkout_completed' + AND TraceId != '' + ORDER BY timestamp DESC + LIMIT 20 + FORMAT JSON + -- builder:service-endpoints:serviceEndpointsSummaryQuery:default [3e379104] SELECT bSpanName AS spanName, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 587e00c20..42ddb023c 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -311,6 +311,29 @@ const productEventsFixtures: ReadonlyArray = [ window, ), }, + // The two directions of the trace ↔ product-event link. Both are + // single-predicate lookups on `TraceId`, so what the sweep is watching for is + // that neither grows a `Map` read or loses its `OrgId`/time bounds. + { + module: "product-events", + name: "productEventsForTraceQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventsForTraceQuery({ limit: 50 }), { + ...window, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }), + }, + { + module: "product-events", + name: "productEventTraceSamplesQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventTraceSamplesQuery({ limit: 20 }), { + ...window, + eventName: "checkout_completed", + }), + }, ] export const builderFixtures: ReadonlyArray = [ diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index e63cddba8..a5b97a3f2 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -189,6 +189,8 @@ export { productEventsFunnelBreakdownRowSchema, productEventNamesQuery, productEventNamesRowSchema, + productEventsForTraceQuery, + productEventTraceSamplesQuery, ProductEventsFunnelError, FUNNEL_MAX_STEPS, FUNNEL_BREAKDOWN_MAX_GROUPS, @@ -196,6 +198,10 @@ export { type FunnelKeyBy, type FunnelSessionDimension, type FunnelBreakdownBy, + type ProductEventsForTraceOpts, + type ProductEventForTraceOutput, + type ProductEventTraceSamplesOpts, + type ProductEventTraceSampleOutput, type ProductEventsFunnelOpts, type ProductEventsFunnelOutput, type ProductEventsFunnelBreakdownOpts, diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index eceedfcea..d61bbbf35 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -694,3 +694,101 @@ export function productEventNamesQuery( .limit(limit) .format("JSON") } + +// Trace ↔ product event linking. An annotated span lands in `product_events` as +// a `Source = 'trace'` row carrying its `TraceId`/`SpanId` (migration 0028 / +// `product_events_traces_mv`); these two queries walk that link in each +// direction. Both filter `TraceId` directly so `idx_trace_id` prunes. +// +// No declared `rowSchema`: every column is a plain String or the Map the builder +// already derives, so a declared copy would only drift. + +export interface ProductEventForTraceOutput { + readonly timestamp: string + readonly eventName: string + readonly spanId: string + readonly serviceName: string + readonly userId: string + readonly groupId: string + readonly visitorId: string + readonly sessionId: string + readonly attributes: Record +} + +export interface ProductEventsForTraceOpts { + /** Default 50 — a single trace producing more than this is pathological. */ + readonly limit?: number +} + +/** + * The product events one trace produced, oldest first. The caller's time window + * is what keeps this off every retained partition; `Source` is not filtered + * because only the trace projection writes a `TraceId`. + */ +export function productEventsForTraceQuery( + opts: ProductEventsForTraceOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + timestamp: $.Timestamp, + eventName: $.EventName, + spanId: $.SpanId, + serviceName: $.ServiceName, + userId: $.UserId, + groupId: $.GroupId, + visitorId: $.VisitorId, + sessionId: $.SessionId, + attributes: $.Attributes, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.TraceId.eq(param.string("traceId")), + ]) + .orderBy(["timestamp", "asc"], ["spanId", "asc"]) + .limit(opts.limit ?? 50) + .format("JSON") +} + +export interface ProductEventTraceSampleOutput { + readonly traceId: string + readonly spanId: string + readonly timestamp: string + readonly serviceName: string + readonly userId: string + readonly visitorId: string +} + +export interface ProductEventTraceSamplesOpts { + /** Default 20. */ + readonly limit?: number +} + +/** + * Recent traces behind one event name, newest first. `TraceId != ''` rather + * than `Source = 'trace'`: same set, and a row with no trace id is not a sample. + */ +export function productEventTraceSamplesQuery( + opts: ProductEventTraceSamplesOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + traceId: $.TraceId, + spanId: $.SpanId, + timestamp: $.Timestamp, + serviceName: $.ServiceName, + userId: $.UserId, + visitorId: $.VisitorId, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.EventName.eq(param.string("eventName")), + $.TraceId.neq(""), + ]) + .orderBy(["timestamp", "desc"]) + .limit(opts.limit ?? 20) + .format("JSON") +} diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 93a59cf4d..b2d38b371 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -805,6 +805,13 @@ export const ProductEvents = table("product_events", { ServiceName: T.string, // track() props. Attributes: T.map(T.string, T.string), + // The trace this event was derived from — non-empty only on Source='trace' + // rows, i.e. spans the customer annotated with `maple.product_event.name`. + // The link in both directions: trace view → its product events, funnel row → + // the trace that performed the step. + TraceId: T.string, + // The annotated span within TraceId. '' on every other source. + SpanId: T.string, }) // (VisitorId, UserId) pairs observed together on a session_replays row. diff --git a/packages/query-engine/src/registry/product-events.ts b/packages/query-engine/src/registry/product-events.ts index c11545d34..7415ee062 100644 --- a/packages/query-engine/src/registry/product-events.ts +++ b/packages/query-engine/src/registry/product-events.ts @@ -1,7 +1,9 @@ import type { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import * as CH from "../ch" import { timeRangeCache } from "../runtime/query-engine" @@ -81,3 +83,33 @@ export const productEventNames = defineQuery({ { orgId, startTime: payload.startTime, endTime: payload.endTime }, ), }) + +// The trace ↔ product-event link, both directions. `list` profile because each is +// a bloom-filter point lookup, and a flat 60s rather than `timeRangeCache` +// because the answer does not widen with the range asked about. + +export const productEventsForTrace = defineQuery({ + id: "productEventsForTrace", + profile: "list", + cache: 60, + compile: (payload: ProductEventsForTraceRequest, orgId: string) => + CH.compile(CH.productEventsForTraceQuery({ limit: payload.limit ?? 50 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + traceId: payload.traceId, + }), +}) + +export const productEventTraceSamples = defineQuery({ + id: "productEventTraceSamples", + profile: "list", + cache: 60, + compile: (payload: ProductEventTraceSamplesRequest, orgId: string) => + CH.compile(CH.productEventTraceSamplesQuery({ limit: payload.limit ?? 20 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + eventName: payload.eventName, + }), +}) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index 0b0185c13..5a3601960 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -56,7 +56,13 @@ import { makeTimeRangeCachePolicy, timeRangeCache } from "../runtime/query-engin import { defineQuery } from "./query-definition" export { logsCount, logsTimeseries } from "./logs" -export { productEventsFunnel, productEventsFunnelBreakdown, productEventNames } from "./product-events" +export { + productEventsFunnel, + productEventsFunnelBreakdown, + productEventNames, + productEventsForTrace, + productEventTraceSamples, +} from "./product-events" /** * Declarative compile, execution, and cache policy. Handlers retain response