diff --git a/apps/alerting/src/scheduled.ts b/apps/alerting/src/scheduled.ts index 65cf78e90..ab32554e8 100644 --- a/apps/alerting/src/scheduled.ts +++ b/apps/alerting/src/scheduled.ts @@ -12,6 +12,7 @@ import { AlertRuntime, AlertRulesService, AlertsService, + AuditLogService, AnomalyDetectionService, BucketCacheService, CacheBackendLive, @@ -147,7 +148,13 @@ export const buildLayer = (env: AlertingWorkerEnv) => { const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(BaseLive)) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(Layer.mergeAll(BaseLive, ErrorActorsServiceLive)), + Layer.provide( + Layer.mergeAll( + BaseLive, + ErrorActorsServiceLive, + AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)), + ), + ), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer.pipe(Layer.provide(BaseLive)) diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 267d9a9d9..52a804268 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -249,6 +249,8 @@ const makeWorkerBindings = ({ vcsSyncQueueName, planetScaleWebhookQueue, planetScaleWebhookQueueName, + auditEventsQueue, + auditEventsQueueName, }: { stage: MapleStage mapleDb: Cloudflare.Hyperdrive.Connection | undefined @@ -258,6 +260,8 @@ const makeWorkerBindings = ({ vcsSyncQueueName: string planetScaleWebhookQueue: Cloudflare.Queues.Queue planetScaleWebhookQueueName: string + auditEventsQueue: Cloudflare.Queues.Queue + auditEventsQueueName: string }) => ({ // Ref stages attach MAPLE_DB via `bindMapleDbRef` below. ...(mapleDb ? { MAPLE_DB: mapleDb } : undefined), @@ -279,6 +283,8 @@ const makeWorkerBindings = ({ VCS_SYNC_QUEUE_NAME: vcsSyncQueueName, PLANETSCALE_WEBHOOK_QUEUE: planetScaleWebhookQueue, PLANETSCALE_WEBHOOK_QUEUE_NAME: planetScaleWebhookQueueName, + AUDIT_EVENTS_QUEUE: auditEventsQueue, + AUDIT_EVENTS_QUEUE_NAME: auditEventsQueueName, // Long-running schema-apply: chunks heavy backfill migrations across durable // steps so they never hit the Worker request budget. Class is exported from // src/worker.ts. The first Workflow arg IS the physical workflow name; the @@ -389,6 +395,15 @@ export const createMapleApi = ({ const planetScaleWebhookQueue = yield* Cloudflare.Queues.Queue("planetscale-webhooks", { name: planetScaleWebhookQueueName, }) + const auditEventsQueueName = resolveWorkerName("audit-events", stage) + const auditEventsQueue = yield* Cloudflare.Queues.Queue("audit-events", { + name: auditEventsQueueName, + }) + // Parking lot for audit entries that exhausted their retries. Deliberately + // has no consumer: an entry landing here is a lost audit record, and the + // point is that it survives for inspection instead of being dropped. + const auditEventsDlqName = resolveWorkerName("audit-events-dlq", stage) + yield* Cloudflare.Queues.Queue("audit-events-dlq", { name: auditEventsDlqName }) const worker = (yield* Cloudflare.Worker("api", { name: resolveWorkerName("api", stage), @@ -434,6 +449,8 @@ export const createMapleApi = ({ vcsSyncQueueName, planetScaleWebhookQueue, planetScaleWebhookQueueName, + auditEventsQueue, + auditEventsQueueName, }), ...configuredEnv, ...devEnv, @@ -463,6 +480,21 @@ export const createMapleApi = ({ maxWaitTimeMs: 5000, }, }) + // Audit entries tolerate a few seconds of delivery latency; batch wider and + // wait longer so one insert round-trip covers many entries. + yield* Cloudflare.Queues.Consumer("audit-events-consumer", { + queueId: auditEventsQueue.queueId, + scriptName: worker.workerName, + // `maxRetries` must stay in sync with AUDIT_EVENTS_MAX_RETRIES in + // audit-events-runtime.ts, which logs the drop on the final attempt. + deadLetterQueue: auditEventsDlqName, + settings: { + batchSize: 25, + maxConcurrency: 2, + maxRetries: 5, + maxWaitTimeMs: 5000, + }, + }) return worker }) diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts index b039942af..5946139e4 100644 --- a/apps/api/src/alerting.ts +++ b/apps/api/src/alerting.ts @@ -4,6 +4,7 @@ export { AlertDestinationsService } from "./services/alerts/AlertDestinationsSer export { AlertReadModelsService } from "./services/alerts/AlertReadModelsService" export { AlertRulesService } from "./services/alerts/AlertRulesService" export { AnomalyDetectionService } from "./services/alerts/AnomalyDetectionService" +export { AuditLogService } from "./services/audit/AuditLogService" export { BucketCacheService } from "@maple/query-engine/caching" export { CacheBackendLive } from "@/platform/CacheBackendLive" export { CloudflareAnalyticsService } from "./services/integrations/CloudflareAnalyticsService" diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts new file mode 100644 index 000000000..8ce050216 --- /dev/null +++ b/apps/api/src/audit-events-runtime.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/primitives" +import type { AuditLogRow } from "@maple/domain/tinybird" +import { WarehouseUpstreamError } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" +import { processAuditEventsBatch } from "./audit-events-runtime" +import { makeWarehouseServiceStub } from "@/routes/v2/v2-test-support" +import { AuditLogEvent, encodeAuditLogEventSync } from "./services/audit/audit-event" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const ORG = asOrgId("org_audit_consumer_test") +const OTHER_ORG = asOrgId("org_audit_consumer_other") + +const event = (id: string, orgId: OrgId = ORG) => + encodeAuditLogEventSync( + new AuditLogEvent({ + orgId, + id: Schema.decodeUnknownSync(AuditLogEvent.fields.id)(id), + actorType: "user", + source: "dashboard", + action: "dashboard.created", + outcome: "allowed", + occurredAtMs: 1_700_000_000_000, + }), + ) + +/** One queue message, recording which terminal call the consumer made on it. */ +const message = (body: unknown, attempts: number) => { + const calls: string[] = [] + return { + message: { + body, + attempts, + ack: () => calls.push("ack"), + retry: () => calls.push("retry"), + }, + calls, + } +} + +const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => + ({ messages: messages.map((entry) => entry.message) }) as never + +/** A warehouse whose `ingest` is interrupted — a deploy tearing the isolate down. */ +const interruptedWarehouse = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ ingest: () => Effect.interrupt }), +) + +/** A warehouse whose `ingest` dies rather than failing — an unexpected defect. */ +const dyingWarehouse = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: () => Effect.die(new Error("ingest exploded")), + }), +) + +/** A warehouse whose `ingest` records each call, or fails every call. */ +const warehouse = (fail = false) => { + const written: Array<{ orgId: string; rows: ReadonlyArray }> = [] + const layer = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: (tenant, _datasource, rows) => + fail + ? Effect.fail( + new WarehouseUpstreamError({ message: "tinybird down", pipeName: "audit_log", cause: new Error("down") }), + ) + : Effect.sync(() => { + // SAFETY: this stub only ever receives the audit datasource's rows. + written.push({ orgId: tenant.orgId, rows: rows as ReadonlyArray }) + }), + }), + ) + return { written, layer } +} + +describe("processAuditEventsBatch", () => { + it.effect("writes well-formed events through ingest, one batch per org, and acks them", () => + Effect.gen(function* () { + const store = warehouse() + const first = message(event("11111111-1111-4111-8111-111111111111"), 1) + const second = message(event("22222222-2222-4222-8222-222222222222"), 1) + const other = message(event("33333333-3333-4333-8333-333333333333", OTHER_ORG), 1) + yield* processAuditEventsBatch(batchOf(first, second, other)).pipe(Effect.provide(store.layer)) + + expect(first.calls).toEqual(["ack"]) + expect(second.calls).toEqual(["ack"]) + expect(other.calls).toEqual(["ack"]) + expect(store.written.map((write) => [write.orgId, write.rows.length]).sort()).toEqual([ + [OTHER_ORG, 1], + [ORG, 2], + ]) + expect(store.written.flatMap((write) => write.rows).every((row) => row.Action === "dashboard.created")).toBe( + true, + ) + }), + ) + + // Cloudflare routes a message to the DLQ only when the consumer retries it + // past `max_retries`. Acking on the final attempt would discard the entry + // instead, which is exactly the silent drop this branch exists to prevent. + it.effect("retries a failed write on the final attempt so the message reaches the DLQ", () => + Effect.gen(function* () { + const exhausted = message(event("44444444-4444-4444-8444-444444444444"), 6) + yield* processAuditEventsBatch(batchOf(exhausted)).pipe(Effect.provide(warehouse(true).layer)) + expect(exhausted.calls).toEqual(["retry"]) + }), + ) + + it.effect("retries every message of a failed org batch while attempts remain", () => + Effect.gen(function* () { + const a = message(event("55555555-5555-4555-8555-555555555555"), 2) + const b = message(event("66666666-6666-4666-8666-666666666666"), 2) + yield* processAuditEventsBatch(batchOf(a, b)).pipe(Effect.provide(warehouse(true).layer)) + expect(a.calls).toEqual(["retry"]) + expect(b.calls).toEqual(["retry"]) + }), + ) + + // A typed failure retries; so must a defect. Catching only the failure + // channel would let an unexpected throw escape the consumer, and Cloudflare + // treats a consumer that neither acked nor retried as a retry anyway — but + // silently, with no log and no DLQ accounting. + it.effect("retries when the write dies instead of failing", () => + Effect.gen(function* () { + const defect = message(event("77777777-7777-4777-8777-777777777777"), 2) + yield* processAuditEventsBatch(batchOf(defect)).pipe(Effect.provide(dyingWarehouse)) + expect(defect.calls).toEqual(["retry"]) + }), + ) + + // Interruption is not a failed attempt. Counting it as one would spend the + // message's retry budget — and eventually route it to the DLQ — for a deploy. + // Unacked is enough: the platform redelivers. + it.effect("does not count an interrupted batch as an attempt", () => + Effect.gen(function* () { + const torn = message(event("88888888-8888-4888-8888-888888888888"), 2) + const exit = yield* processAuditEventsBatch(batchOf(torn)).pipe( + Effect.provide(interruptedWarehouse), + Effect.exit, + ) + expect(exit._tag).toBe("Failure") + expect(torn.calls).toEqual([]) + }), + ) + + // A message that cannot decode will never decode. Retrying only burns the + // attempts that would otherwise carry a recoverable message to the DLQ. + it.effect("acks a malformed message instead of retrying it forever", () => + Effect.gen(function* () { + const store = warehouse() + const malformed = message({ not: "an audit event" }, 1) + const fine = message(event("77777777-7777-4777-8777-777777777777"), 1) + yield* processAuditEventsBatch(batchOf(malformed, fine)).pipe(Effect.provide(store.layer)) + + expect(malformed.calls).toEqual(["ack"]) + expect(fine.calls).toEqual(["ack"]) + expect(store.written).toHaveLength(1) + }), + ) +}) diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts new file mode 100644 index 000000000..1efee2ff9 --- /dev/null +++ b/apps/api/src/audit-events-runtime.ts @@ -0,0 +1,176 @@ +import type { Message, MessageBatch } from "@cloudflare/workers-types" +import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" +import { EdgeCacheService } from "@maple/cache" +import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" +import type { OrgId } from "@maple/domain/primitives" +import { WorkerConfigProviderLayer, workerEnvironmentLayer } from "@maple/infra/worker-runtime" +import { Cause, Clock, Effect, Layer } from "effect" +import { CacheBackendLive } from "@/platform/CacheBackendLive" +import { summarizeCause } from "@/platform/describe-cause" +import { layerPg } from "@/platform/DatabasePgLive" +import { Env } from "@/platform/Env" +import { systemTenant } from "@/services/alerts/system-tenant" +import { AUDIT_LOG_DATASOURCE } from "@/services/audit/AuditLogService" +import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "@/services/integrations/TinybirdOrgTokenService" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { type AuditLogEvent, auditEventToRow, decodeAuditLogEvent } from "./services/audit/audit-event" + +const telemetry = MapleCloudflareSDK.make({ + serviceName: "maple-api", + serviceNamespace: "core", + repositoryUrl: "https://github.com/MapleTechLabs/maple", + anticipatedErrorIdentifiers: [...ANTICIPATED_ERROR_IDENTIFIERS], +}) + +/** + * The consumer writes through `WarehouseQueryService.ingest`, which pins every + * write to the managed Tinybird pipeline. The service's read-side dependencies + * (org ClickHouse settings, the per-org JWT minter) come along because the + * layer requires them, not because a write ever consults them. + */ +export const buildAuditEventsLayer = (_env: Record) => { + const EnvLive = Env.layer.pipe(Layer.provide(WorkerConfigProviderLayer)) + const DatabaseLive = layerPg.pipe(Layer.provide(workerEnvironmentLayer)) + const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive)) + const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, DatabaseLive, EdgeCacheServiceLive)), + ) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), + ) + return WarehouseQueryServiceLive.pipe( + Layer.provideMerge(telemetry.layer), + Layer.provideMerge(workerEnvironmentLayer), + Layer.provideMerge(WorkerConfigProviderLayer), + ) +} + +export const flushAuditEventsTelemetry = (env: Record) => telemetry.flush(env) + +/** + * Must match `maxRetries` on the audit-events consumer in `alchemy.run.ts`. + * Cloudflare routes the message to the DLQ after this many retries without + * telling us; the check below is what makes the hand-off visible in logs at + * the moment it happens. + */ +const AUDIT_EVENTS_MAX_RETRIES = 5 + +/** + * Best-effort identity for the exhaustion log. The body reached us as queue + * JSON and may be anything at all, so these read defensively rather than + * decoding — a drop must still be reported when the payload is the problem. + */ +const auditEventField = (body: unknown, field: string): string => { + if (typeof body !== "object" || body === null || !(field in body)) return "" + // SAFETY: `field in body` established the key exists on this object. + const value = (body as Record)[field] + return typeof value === "string" ? value : "" +} +const auditEventOrgId = (body: unknown) => auditEventField(body, "orgId") +const auditEventAction = (body: unknown) => auditEventField(body, "action") + +interface DecodedMessage { + readonly message: Message + readonly event: AuditLogEvent +} + +/** + * Retrying past the limit is what hands the message to the DLQ; acking there + * would silently discard it instead. + */ +const retryOrExhaust = (message: Message, cause: unknown) => { + const isFinalAttempt = message.attempts > AUDIT_EVENTS_MAX_RETRIES + return Effect.annotateCurrentSpan({ + "audit.queue.message.outcome": isFinalAttempt ? "exhausted_dlq" : "retry", + }).pipe( + Effect.flatMap(() => + isFinalAttempt + ? Effect.logError("Audit event exhausted retries; routed to dead letter queue").pipe( + Effect.annotateLogs({ + attempt: message.attempts, + orgId: auditEventOrgId(message.body), + action: auditEventAction(message.body), + error: String(cause), + }), + ) + : Effect.logWarning("Audit event write failed; retrying").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(cause) }), + ), + ), + Effect.flatMap(() => Effect.sync(() => message.retry())), + ) +} + +/** + * Audit events queue consumer: lowers each event to its `audit_log` row and + * writes one batch per org through the managed ingest pipeline. The table is a + * ReplacingMergeTree on the entry id, so queue redelivery collapses at merge + * time; write failures retry through the queue's policy and, once exhausted, + * land in `audit-events-dlq` rather than disappearing. + */ +export const processAuditEventsBatch = (batch: MessageBatch) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const now = yield* Clock.currentTimeMillis + + const decoded: Array = [] + for (const message of batch.messages) { + const event = yield* decodeAuditLogEvent(message.body).pipe( + Effect.matchEffect({ + // Undecodable now means undecodable on every redelivery, so retrying + // only burns attempts. Acked, but at Error: an audit entry that + // never reaches a row is lost evidence, not routine noise. + onFailure: (error) => + Effect.logError("Discarding malformed audit event queue message").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(error) }), + Effect.flatMap(() => Effect.sync(() => message.ack())), + Effect.as(undefined), + ), + onSuccess: (event) => Effect.succeed(event), + }), + ) + if (event !== undefined) decoded.push({ message, event }) + } + + // One `ingest` per org so the write span names the tenant it belongs to. + const byOrg = new Map>() + for (const entry of decoded) { + const group = byOrg.get(entry.event.orgId) + if (group === undefined) byOrg.set(entry.event.orgId, [entry]) + else group.push(entry) + } + + yield* Effect.forEach( + byOrg, + ([orgId, group]) => + warehouse + .ingest( + systemTenant(orgId), + AUDIT_LOG_DATASOURCE, + group.map(({ event }) => auditEventToRow(event, now)), + ) + .pipe( + Effect.flatMap(() => + Effect.sync(() => { + for (const { message } of group) message.ack() + }), + ), + Effect.withSpan("auditEvents.writeOrgBatch", { attributes: { orgId, rows: group.length } }), + // A failure or a defect is a failed attempt and retries. Interruption + // is not: an interrupted batch (a deploy, an isolate torn down) + // counted as an attempt would push messages toward the DLQ for + // something that never failed. Re-raised, it leaves the batch + // unacked and the platform redelivers it. + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.forEach(group, ({ message }) => retryOrExhaust(message, summarizeCause(cause)), { + discard: true, + }), + ), + ), + { concurrency: 3, discard: true }, + ) + }).pipe(Effect.withSpan("auditEvents.processBatch")) diff --git a/apps/api/src/mcp/app.ts b/apps/api/src/mcp/app.ts index 874cd0652..d670f7306 100644 --- a/apps/api/src/mcp/app.ts +++ b/apps/api/src/mcp/app.ts @@ -10,6 +10,8 @@ import { InstructionsResource } from "./resources/instructions" import { sessionStore } from "./lib/session-store" import type { McpToolExecutor } from "./dispatcher" import { CurrentMcpRequestTenant, CurrentMcpTenant, resolveHttpMcpTenant } from "./lib/query-warehouse" +import { type AuditActorInfo, CurrentAuditActor } from "@/services/auth/audit-actor" +import { INTERNAL_SERVICE_PREFIX } from "./lib/resolve-tenant" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { @@ -98,6 +100,23 @@ const mcpUnavailable = () => ), ) +/** + * Which credential an MCP request presented, as far as the transport can tell. + * Mirrors the branches in `resolveMcpTenantContext`: an internal service token + * is Maple acting on its own behalf, any other bearer is an API key or OAuth + * token, and no bearer at all means a forwarded dashboard session. + */ +const mcpAuditActor = (headers: Record): AuditActorInfo => { + const authorization = headers["authorization"] ?? headers["Authorization"] + if (authorization?.toLowerCase().startsWith("bearer ") !== true) { + return { type: "user", source: "mcp" } + } + const bearer = authorization.slice("bearer ".length).trim() + return bearer.startsWith(INTERNAL_SERVICE_PREFIX) + ? { type: "system", source: "system" } + : { type: "api_key", source: "mcp" } +} + // Wording mirrors the v2 envelope's `V2RateLimited`; the body stays in this // surface's `{ error, message }` shape like the 401/503 responses above. const mcpRateLimited = () => @@ -141,6 +160,12 @@ const McpAuthorizationMiddleware = HttpRouter.middleware<{ provides: CurrentMcpT } return yield* Effect.provideService(httpEffect, CurrentMcpTenant, tenant).pipe( Effect.provideService(CurrentMcpRequestTenant, tenant), + // Without this an MCP mutation reads the reference's `undefined` + // default and is audited as a dashboard session. The credential + // kind is all this layer can see — `resolveMcpTenantContext` + // returns the tenant, not the key it resolved — so the key id is + // deliberately absent rather than guessed. + Effect.provideService(CurrentAuditActor, mcpAuditActor(request.headers)), ) }), ), diff --git a/apps/api/src/mcp/dispatcher.test.ts b/apps/api/src/mcp/dispatcher.test.ts index 51ff778fc..0c8fde2fa 100644 --- a/apps/api/src/mcp/dispatcher.test.ts +++ b/apps/api/src/mcp/dispatcher.test.ts @@ -6,6 +6,7 @@ import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "./expected-failures" import { mapleToolCatalog, toInputSchema } from "./tools/registry" import type { McpToolRuntimeRequirements } from "./tools/runtime-requirements" import type { TenantContext } from "@/services/auth/tenant-context" +import { AuditLogService, makeMemoryAuditLog } from "@/services/audit/AuditLogService" const TENANT: TenantContext = { orgId: "org_test" as TenantContext["orgId"], @@ -16,7 +17,8 @@ const TENANT: TenantContext = { // These cases stop at registry lookup/schema decoding, before a tool service is read. const makeValidationExecutor = McpToolExecutor.make.pipe( - Effect.provide(Context.empty() as Context.Context), + // Every tool call is audited, so the executor needs the audit service even here. + Effect.provide(Context.make(AuditLogService, makeMemoryAuditLog()) as Context.Context), ) const makeRecordingTracer = () => { diff --git a/apps/api/src/mcp/dispatcher.ts b/apps/api/src/mcp/dispatcher.ts index 687a5340f..e7e306730 100644 --- a/apps/api/src/mcp/dispatcher.ts +++ b/apps/api/src/mcp/dispatcher.ts @@ -7,6 +7,7 @@ import type { McpToolRuntimeRequirements } from "./tools/runtime-requirements" import { CurrentMcpTenant } from "./lib/query-warehouse" import { recordExpectedMcpFailure } from "./expected-failures" import type { TenantContext } from "@/services/auth/tenant-context" +import { recordMcpToolAudit } from "@/services/audit/audit-access" /** * Built on first use, not at module scope. @@ -172,10 +173,20 @@ export class McpToolExecutor extends Context.Service Layer.succeed(WarehouseQueryService, stub) +// `runRawSql` records `telemetry.sql_executed` on every path, rejections +// included, so the audit service is part of every harness here — the in-memory +// one, since what is asserted below is the SQL, not the audit row. +const provide = (stub: WarehouseQueryServiceApi) => + Layer.merge(Layer.succeed(WarehouseQueryService, stub), AuditLogService.layerMemory) const range = { startTime: "2026-04-01 00:00:00", endTime: "2026-04-01 01:00:00" } diff --git a/apps/api/src/mcp/lib/run-raw-sql.ts b/apps/api/src/mcp/lib/run-raw-sql.ts index c4d0daf2d..963f62951 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.ts @@ -5,6 +5,7 @@ import { computeBucketSecondsForRange } from "@maple/query-engine" import { makeExecuteRawSql } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import type { TenantContext } from "@/services/auth/tenant-context" +import { describeFailure, recordRawSqlAudit } from "@/services/audit/audit-access" /** * `$__interval_s` when the caller doesn't pin `granularitySeconds`. @@ -42,6 +43,15 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput const executeRawSql = makeExecuteRawSql( warehouse, ) + const audit = (result: Parameters[0]["result"]) => + recordRawSqlAudit({ + tenant: input.tenant, + sql: input.sql, + context: "mcp.run_sql", + startTime: input.startTime, + endTime: input.endTime, + result, + }) return yield* executeRawSql(input.tenant, { sql: input.sql, orgId: input.tenant.orgId, @@ -50,5 +60,15 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput granularitySeconds: input.granularitySeconds, workload: "interactive", context: "mcp.run_sql", - }) + }).pipe( + // Every statement is audited, however it ended: a refused one as `denied`. + Effect.tap((result) => audit({ _tag: "rows", rowCount: result.rowCount })), + Effect.tapError((error) => + audit( + error._tag === "@maple/http/errors/RawSqlValidationError" + ? { _tag: "rejected", reason: error.message } + : { _tag: "failed", error: describeFailure(error) }, + ), + ), + ) }) diff --git a/apps/api/src/mcp/tools/register-agent.ts b/apps/api/src/mcp/tools/register-agent.ts index be1f258cb..5e0006543 100644 --- a/apps/api/src/mcp/tools/register-agent.ts +++ b/apps/api/src/mcp/tools/register-agent.ts @@ -8,6 +8,7 @@ import { import { Effect, Option, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "@/services/errors/ErrorActorsService" const decodeStringArray = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Array(Schema.String))) @@ -58,6 +59,16 @@ export function registerRegisterAgentTool(server: McpToolRegistrar) { ), ) + const audit = yield* AuditLogService + yield* audit.record({ + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "mcp", + action: "agent.registered", + resourceId: actor.id, + metadata: { name: actor.agentName ?? name }, + }) + const lines = [ `## Agent registered`, `- Actor ID: ${actor.id}`, diff --git a/apps/api/src/mcp/tools/runtime-requirements.ts b/apps/api/src/mcp/tools/runtime-requirements.ts index baf795ab8..8b4e1f3e3 100644 --- a/apps/api/src/mcp/tools/runtime-requirements.ts +++ b/apps/api/src/mcp/tools/runtime-requirements.ts @@ -1,3 +1,4 @@ +import type { AuditLogService } from "@/services/audit/AuditLogService" import type { AlertsService } from "@/services/alerts/AlertsService" import type { AlertReadModelsService } from "@/services/alerts/AlertReadModelsService" import type { AlertRulesService } from "@/services/alerts/AlertRulesService" @@ -22,6 +23,7 @@ import type { CurrentMcpTenant } from "../lib/query-warehouse" */ export type McpToolRuntimeRequirements = | AlertsService + | AuditLogService | AlertReadModelsService | AlertRulesService | DashboardPersistenceService diff --git a/apps/api/src/platform/time.ts b/apps/api/src/platform/time.ts index e4dae4216..e4b74ebca 100644 --- a/apps/api/src/platform/time.ts +++ b/apps/api/src/platform/time.ts @@ -32,3 +32,4 @@ export function dateToMs(date: Date | null | undefined): number | null export function dateToMs(date: Date | null | undefined): number | null { return date === null || date === undefined ? null : date.getTime() } + diff --git a/apps/api/src/queue-dispatch.ts b/apps/api/src/queue-dispatch.ts index 77f3e320e..44502e140 100644 --- a/apps/api/src/queue-dispatch.ts +++ b/apps/api/src/queue-dispatch.ts @@ -1,4 +1,4 @@ -export type WorkerQueueKind = "planetscale-webhook" | "vcs-sync" | "unknown" +export type WorkerQueueKind = "planetscale-webhook" | "vcs-sync" | "audit-events" | "unknown" export const classifyWorkerQueue = (queueName: string, env: Record): WorkerQueueKind => { if ( @@ -10,5 +10,8 @@ export const classifyWorkerQueue = (queueName: string, env: Record[0]["result"]) => + recordRawSqlAudit({ + tenant, + sql: payload.sql, + context: "rawSql", + startTime: payload.startTime, + endTime: payload.endTime, + result, + }) const result = yield* mapExecError( executeRawSql(tenant, { sql: payload.sql, @@ -2087,7 +2097,17 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query granularitySeconds, workload: "interactive", context: "rawSql", - }), + }).pipe( + // Every statement is audited, however it ended: a refused one as `denied`. + Effect.tap((executed) => audit({ _tag: "rows", rowCount: executed.rowCount })), + Effect.tapError((error) => + audit( + error._tag === "@maple/http/errors/RawSqlValidationError" + ? { _tag: "rejected", reason: error.message } + : { _tag: "failed", error: describeFailure(error) }, + ), + ), + ), "rawSql query failed", ) diff --git a/apps/api/src/routes/v1/org-clickhouse-settings.http.ts b/apps/api/src/routes/v1/org-clickhouse-settings.http.ts index 79e1f2857..6d43e4b8f 100644 --- a/apps/api/src/routes/v1/org-clickhouse-settings.http.ts +++ b/apps/api/src/routes/v1/org-clickhouse-settings.http.ts @@ -1,6 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant, MapleApi } from "@maple/domain/http" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( @@ -20,7 +21,18 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("upsert", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.upsert(tenant.orgId, tenant.userId, tenant.roles, payload) + const updated = yield* service.upsert( + tenant.orgId, + tenant.userId, + tenant.roles, + payload, + ) + // URL/user/database identify the connection; the password in the + // payload is write-only and never reaches an audit row. + yield* recordHttpAudit("warehouse_settings.updated", { + metadata: { url: payload.url, user: payload.user, database: payload.database }, + }) + return updated }), ) .handle("schemaDiff", () => @@ -32,7 +44,9 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("applySchema", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.applySchema(tenant.orgId, tenant.userId, tenant.roles) + const applied = yield* service.applySchema(tenant.orgId, tenant.userId, tenant.roles) + yield* recordHttpAudit("warehouse_settings.schema_applied") + return applied }), ) .handle("applySchemaStatus", () => @@ -50,7 +64,9 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("delete", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.delete(tenant.orgId, tenant.roles) + const deleted = yield* service.delete(tenant.orgId, tenant.roles) + yield* recordHttpAudit("warehouse_settings.deleted") + return deleted }), ) }), diff --git a/apps/api/src/routes/v1/organizations.http.ts b/apps/api/src/routes/v1/organizations.http.ts index 231b7ce75..7b8dfb4b5 100644 --- a/apps/api/src/routes/v1/organizations.http.ts +++ b/apps/api/src/routes/v1/organizations.http.ts @@ -1,6 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant, MapleApi } from "@maple/domain/http" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrganizationService } from "@/services/org/OrganizationService" export const HttpOrganizationsLive = HttpApiBuilder.group(MapleApi, "organizations", (handlers) => @@ -10,7 +11,12 @@ export const HttpOrganizationsLive = HttpApiBuilder.group(MapleApi, "organizatio return handlers.handle("delete", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* organizationService.delete(tenant.orgId, tenant.roles) + const deleted = yield* organizationService.delete(tenant.orgId, tenant.roles) + // Recorded after the fact so a refused delete cannot leave an entry + // claiming the org is gone. The entry outlives the org: the audit + // log is never cascaded, which is the point of a trail. + yield* recordHttpAudit("organization.deleted") + return deleted }), ) }), diff --git a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts index 2cf2cc363..201d2daee 100644 --- a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts +++ b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts @@ -30,6 +30,7 @@ import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglit import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -174,6 +175,7 @@ const makeHarness = () => { Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/alert-destinations.http.ts b/apps/api/src/routes/v2/alert-destinations.http.ts index ea2660a9d..a33585d6d 100644 --- a/apps/api/src/routes/v2/alert-destinations.http.ts +++ b/apps/api/src/routes/v2/alert-destinations.http.ts @@ -1,5 +1,6 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import type { AlertDestinationDocument, AlertDestinationUpdateRequest } from "@maple/domain/http" +import { auditDiff } from "./audit-changes" import { CurrentTenant, DiscordAlertDestinationConfig, @@ -20,6 +21,7 @@ import type { } from "@maple/domain/http/v2" import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertDestinationsService } from "@/services/alerts/AlertDestinationsService" const toV2Destination = (doc: AlertDestinationDocument): V2AlertDestination => ({ @@ -191,6 +193,37 @@ const toUpdateRequest = (params: V2AlertDestinationUpdateParams): AlertDestinati } } +/** Credential-bearing config keys; their values must never reach the audit row. */ +/** + * The three update fields a destination document echoes back. Everything else + * an update can carry is either a credential or a provider-side handle the + * document never returns, so the diff can only record that it was touched. + */ +const destinationAuditView = (doc: AlertDestinationDocument | undefined) => ({ + name: doc?.name, + enabled: doc?.enabled, + member_user_ids: doc?.memberUserIds, +}) + +export const destinationAuditDiff = auditDiff({ + fields: ["name", "enabled", "member_user_ids"], + // A webhook URL is a credential: Discord's carries the token in the path, and + // a plain webhook's can carry one in userinfo or query. + writeOnly: ["integration_key", "signing_secret", "url", "webhook_url", "bot_token"], + // Provider-side handles. Not secret, but not readable back off the document + // either — recorded as touched so the change is not invisible. + opaque: [ + "channel_id", + "channel_name", + "chat_id", + "hazel_organization_id", + "hazel_organization_name", + "hazel_organization_logo_url", + "hazel_channel_id", + "hazel_channel_name", + ], +}) + export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "alertDestinations", (handlers) => Effect.gen(function* () { const destinations = yield* AlertDestinationsService @@ -241,19 +274,38 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale toCreateRequest(payload), ) + yield* recordHttpAudit("alert_destination.created", { + resourceId: created.id, + metadata: { name: created.name, type: created.type }, + }) + return toV2DestinationMutation(created) }), ) .handle("update", ({ params, payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + const request = toUpdateRequest(payload) + const existing = yield* destinations.listDestinations(tenant.orgId) + const current = existing.destinations.find((doc) => doc.id === params.id) const updated = yield* destinations.updateDestination( tenant.orgId, tenant.userId, tenant.roles, params.id, - toUpdateRequest(payload), + request, + ) + + const changes = destinationAuditDiff( + payload, + destinationAuditView(current), + destinationAuditView(updated), ) + yield* recordHttpAudit("alert_destination.updated", { + resourceId: updated.id, + changes, + metadata: { name: updated.name, type: updated.type }, + }) return toV2DestinationMutation(updated) }), @@ -266,6 +318,9 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale tenant.roles, params.id, ) + yield* recordHttpAudit("alert_destination.deleted", { + resourceId: deleted.id, + }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 73d7e3d84..abfb40c5b 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -19,6 +19,8 @@ import type { import { MapleApiV2, paginateArray, scopeAllows, timestamp, V2ParameterInvalid } from "@maple/domain/http/v2" import { AlertForbiddenError } from "@maple/domain/http" import { Effect, Encoding, Result, Schema } from "effect" +import { auditDiff } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertsService } from "@/services/alerts/AlertsService" import { AlertReadModelsService } from "@/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@/services/alerts/AlertRulesService" @@ -94,6 +96,38 @@ const toV2Rule = (doc: AlertRuleDocument): V2AlertRule => ({ updated_by: doc.updatedBy, }) +/** Update-payload fields diffable through the wire shape (drafts get summarized). */ +const ruleAuditDiff = auditDiff({ + fields: [ + "name", + "notes", + "notification_template", + "enabled", + "severity", + "service_names", + "exclude_service_names", + "environments", + "tags", + "group_by", + "signal_type", + "comparator", + "threshold", + "threshold_upper", + "window_minutes", + "minimum_sample_count", + "consecutive_breaches_required", + "consecutive_healthy_required", + "renotify_interval_minutes", + "apdex_threshold_ms", + "query_builder_draft", + "raw_query_sql", + "raw_query_reducer", + "destination_ids", + ], + // Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. + summarize: { query_builder_draft: "", raw_query_sql: "" }, +}) + const toV2RuleMutationResponse = (doc: AlertRuleDocument): V2AlertRuleMutationResponse => ({ ...toV2Rule(doc), ...(doc.txid !== undefined ? { txid: doc.txid } : undefined), @@ -345,6 +379,11 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules request, ) + yield* recordHttpAudit("alert_rule.created", { + resourceId: created.id, + metadata: { name: created.name }, + }) + return toV2RuleMutationResponse(created) }), ) @@ -361,6 +400,12 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules request, ) + yield* recordHttpAudit("alert_rule.updated", { + resourceId: updated.id, + changes: ruleAuditDiff(payload, toV2Rule(current), toV2Rule(updated)), + metadata: { name: updated.name }, + }) + return toV2RuleMutationResponse(updated) }), ) @@ -368,6 +413,7 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* rules.deleteRule(tenant.orgId, tenant.roles, params.id) + yield* recordHttpAudit("alert_rule.deleted", { resourceId: deleted.id }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 418a76f00..4e84f09ad 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -18,6 +18,7 @@ import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@/platfor import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -164,6 +165,7 @@ const makeHarness = ( Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/anomalies.http.ts b/apps/api/src/routes/v2/anomalies.http.ts index d7e54a64f..bc8d0bac4 100644 --- a/apps/api/src/routes/v2/anomalies.http.ts +++ b/apps/api/src/routes/v2/anomalies.http.ts @@ -15,6 +15,7 @@ import { import { MapleApiV2, paginateOffsetQuery, timestamp } from "@maple/domain/http/v2" import type { V2AnomalyIncident, V2AnomalyIncidentTimeseries, V2AnomalySettings } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" import { AnomalyDetectionService } from "@/services/alerts/AnomalyDetectionService" import { ErrorsService } from "@/services/errors/ErrorsService" @@ -188,6 +189,13 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const incident = yield* anomalies.resolveIncidentManually(tenant.orgId, params.id) + yield* recordHttpAudit("anomaly_incident.resolved", { + resourceId: incident.id, + metadata: { + signal_type: incident.signalType, + service_name: incident.serviceName, + }, + }) return toV2Incident(incident) }), @@ -246,6 +254,10 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", }), ) + yield* recordHttpAudit("anomaly_settings.updated", { + metadata: { enabled: settings.enabled, sensitivity: settings.sensitivity }, + }) + return toV2Settings(settings) }), ) diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index 2e7432231..2b2500e84 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -12,6 +12,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiV2RateLimiter, type RateLimiterApi } from "@/services/auth/ApiV2RateLimiter" import { V2TransportErrorBoundaryLive } from "./error-envelope" import { @@ -67,6 +68,7 @@ const makeHarness = (checkRateLimit: RateLimiterApi["check"] = () => Effect.succ Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(Layer.succeed(ApiV2RateLimiter, { check: checkRateLimit })), Layer.provideMerge(servicesLive), Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS)), diff --git a/apps/api/src/routes/v2/api-keys.http.ts b/apps/api/src/routes/v2/api-keys.http.ts index 581de9352..0e737a3e5 100644 --- a/apps/api/src/routes/v2/api-keys.http.ts +++ b/apps/api/src/routes/v2/api-keys.http.ts @@ -10,6 +10,7 @@ import { } from "@maple/domain/http/v2" import type { V2ApiKey, V2ApiKeyMutationResponse, V2ApiKeyWithSecret } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { requireAdmin } from "@/services/auth/auth" @@ -106,6 +107,10 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha ? { metadataJson: { source: "maple_mcp", roles: [...tenant.roles] } } : undefined), }) + yield* recordHttpAudit("api_key.created", { + resourceId: created.id, + metadata: { name: created.name, kind: created.kind, scopes: created.scopes }, + }) return toV2ApiKeyWithSecret(created) }), ) @@ -117,6 +122,10 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha const rolled = yield* apiKeysService.roll(tenant.orgId, tenant.userId, params.id, { createdByEmail, }) + yield* recordHttpAudit("api_key.rolled", { + resourceId: rolled.id, + metadata: { name: rolled.name, scopes: rolled.scopes }, + }) return toV2ApiKeyWithSecret(rolled) }), ) @@ -132,6 +141,10 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha yield* requireAdmin(tenant.roles, adminOnly("revoke")) } const revoked = yield* apiKeysService.revoke(tenant.orgId, params.id) + yield* recordHttpAudit("api_key.revoked", { + resourceId: revoked.id, + metadata: { name: revoked.name }, + }) return toV2ApiKeyMutationResponse(revoked) }), ) diff --git a/apps/api/src/routes/v2/attribute-mappings.http.ts b/apps/api/src/routes/v2/attribute-mappings.http.ts index 3d70e227f..50eba570f 100644 --- a/apps/api/src/routes/v2/attribute-mappings.http.ts +++ b/apps/api/src/routes/v2/attribute-mappings.http.ts @@ -11,6 +11,8 @@ import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" import type { V2AttributeMapping } from "@maple/domain/http/v2" import { Array as Arr, Effect, Option } from "effect" import { requireAdmin } from "@/services/auth/auth" +import { diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { IngestAttributeMappingService } from "@/services/org/IngestAttributeMappingService" const toV2AttributeMapping = (mapping: IngestAttributeMapping): V2AttributeMapping => ({ @@ -26,6 +28,11 @@ const toV2AttributeMapping = (mapping: IngestAttributeMapping): V2AttributeMappi updated_at: mapping.updatedAt, }) +/** Update-payload fields that are diffable through the wire shape. */ +const mappingAuditKeys: ReadonlyArray< + "name" | "source_context" | "source_key" | "target_key" | "operation" | "enabled" +> = ["name", "source_context", "source_key", "target_key", "operation", "enabled"] + export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "attributeMappings", (handlers) => Effect.gen(function* () { const service = yield* IngestAttributeMappingService @@ -95,6 +102,11 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att }), ) + yield* recordHttpAudit("attribute_mapping.created", { + resourceId: created.id, + metadata: { name: created.name }, + }) + return toV2AttributeMapping(created) }), ) @@ -102,6 +114,7 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* requireMappingAdmin(tenant, "update") + const current = yield* findMapping(tenant.orgId, params.id) const updated = yield* service.update( tenant.orgId, params.id, @@ -125,6 +138,16 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att }), ) + const changes = diffAuditChanges( + pickPresentFields(mappingAuditKeys, payload, toV2AttributeMapping(current)), + pickPresentFields(mappingAuditKeys, payload, toV2AttributeMapping(updated)), + ) + yield* recordHttpAudit("attribute_mapping.updated", { + resourceId: updated.id, + changes, + metadata: { name: updated.name }, + }) + return toV2AttributeMapping(updated) }), ) @@ -133,6 +156,9 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att const tenant = yield* CurrentTenant.Context yield* requireMappingAdmin(tenant, "delete") const deleted = yield* service.delete(tenant.orgId, params.id) + yield* recordHttpAudit("attribute_mapping.deleted", { + resourceId: deleted.id, + }) return { id: deleted.id, object: "attribute_mapping" as const, deleted: true as const } }), diff --git a/apps/api/src/routes/v2/audit-changes.test.ts b/apps/api/src/routes/v2/audit-changes.test.ts new file mode 100644 index 000000000..95fe688d6 --- /dev/null +++ b/apps/api/src/routes/v2/audit-changes.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest" +import { auditDiff, redactAuditUrl } from "./audit-changes" +import { destinationAuditDiff } from "./alert-destinations.http" + +const targetDiff = auditDiff({ + fields: ["name", "url", "enabled", "labels_json"], + summarize: { labels_json: "" }, + redact: { url: redactAuditUrl }, + writeOnly: ["auth_credentials"], +}) + +describe("auditDiff", () => { + it("diffs only the fields the payload carried", () => { + const changes = targetDiff( + { name: "renamed" }, + { name: "before", url: "https://a.test/x", enabled: true, labels_json: "{}" }, + { name: "renamed", url: "https://b.test/y", enabled: false, labels_json: "{}" }, + ) + // `url` and `enabled` moved, but the request did not ask for them. + expect(changes).toEqual({ + fields: ["name"], + before: { name: "before" }, + after: { name: "renamed" }, + }) + }) + + it("returns undefined when a touched field is unchanged", () => { + expect( + targetDiff( + { name: "same" }, + { name: "same", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "same", url: "https://a.test", enabled: true, labels_json: "{}" }, + ), + ).toBeUndefined() + }) + + it("redacts credentials out of a changed URL", () => { + const changes = targetDiff( + { url: "https://user:secret@b.test/m?token=live" }, + { name: "n", url: "https://a.test/m", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://user:secret@b.test/m?token=live", enabled: true, labels_json: "{}" }, + ) + expect(changes?.after["url"]).toBe("https://b.test/m") + expect(JSON.stringify(changes)).not.toContain("secret") + expect(JSON.stringify(changes)).not.toContain("token=live") + }) + + it("summarizes config blobs instead of recording their bodies", () => { + const changes = targetDiff( + { labels_json: '{"team":"infra"}' }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: '{"team":"infra"}' }, + ) + expect(changes).toEqual({ + fields: ["labels_json"], + before: { labels_json: "" }, + after: { labels_json: "" }, + }) + }) + + it("records a write-only field as rotated whenever the payload carries it", () => { + const changes = targetDiff( + { auth_credentials: "hunter2" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + ) + expect(changes).toEqual({ + fields: ["auth_credentials"], + before: { auth_credentials: "" }, + after: { auth_credentials: "" }, + }) + expect(JSON.stringify(changes)).not.toContain("hunter2") + }) + + it("merges a rotated credential into an observable diff", () => { + const changes = targetDiff( + { name: "renamed", auth_credentials: "hunter2" }, + { name: "before", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "renamed", url: "https://a.test", enabled: true, labels_json: "{}" }, + ) + expect(changes?.fields).toEqual(["name", "auth_credentials"]) + expect(changes?.after).toEqual({ name: "renamed", auth_credentials: "" }) + }) +}) + +describe("auditDiff opaque fields", () => { + const knobDiff = auditDiff({ + fields: ["name"], + opaque: ["channel_id"], + writeOnly: ["bot_token"], + }) + + it("records a provider-side handle as touched, not as its value", () => { + const changes = knobDiff({ channel_id: "C0123" }, { name: "n" }, { name: "n" }) + expect(changes).toEqual({ + fields: ["channel_id"], + before: { channel_id: "" }, + after: { channel_id: "" }, + }) + // Not secret, but not ours to echo either — the document never returns it. + expect(JSON.stringify(changes)).not.toContain("C0123") + }) + + it("keeps a knob distinct from a credential in the same request", () => { + const changes = knobDiff( + { name: "renamed", channel_id: "C0123", bot_token: "xoxb-secret" }, + { name: "before" }, + { name: "renamed" }, + ) + expect(changes?.fields).toEqual(["name", "bot_token", "channel_id"]) + expect(changes?.after).toEqual({ + name: "renamed", + bot_token: "", + channel_id: "", + }) + }) + + it("says nothing when the request carried neither", () => { + expect(knobDiff({ name: "same" }, { name: "same" }, { name: "same" })).toBeUndefined() + }) +}) + +describe("destinationAuditDiff", () => { + const view = (over: Record = {}) => ({ + name: "Ops", + enabled: true, + member_user_ids: undefined, + ...over, + }) + + it("diffs the fields a destination document echoes", () => { + const changes = destinationAuditDiff({ enabled: false }, view(), view({ enabled: false })) + expect(changes).toEqual({ fields: ["enabled"], before: { enabled: true }, after: { enabled: false } }) + }) + + it("never records a rotated credential's value", () => { + const changes = destinationAuditDiff({ bot_token: "xoxb-secret" }, view(), view()) + expect(changes?.fields).toEqual(["bot_token"]) + expect(JSON.stringify(changes)).not.toContain("xoxb-secret") + }) + + // A webhook URL is the credential for Discord and can carry one for a plain + // webhook, so it is withheld rather than diffed. + it("treats a webhook URL as a credential", () => { + const changes = destinationAuditDiff({ webhook_url: "https://discord.test/api/webhooks/1/tok" }, view(), view()) + expect(changes?.after).toEqual({ webhook_url: "" }) + }) + + it("records a channel move as touched", () => { + const changes = destinationAuditDiff({ channel_id: "C9", channel_name: "#alerts" }, view(), view()) + expect(changes?.fields).toEqual(["channel_id", "channel_name"]) + expect(changes?.after).toEqual({ channel_id: "", channel_name: "" }) + }) + + // The pre-update document is looked up from a list; if it were missing, the + // diff must still record what the request changed rather than nothing. + it("still records a change when the previous document is unknown", () => { + const unknown = { name: undefined, enabled: undefined, member_user_ids: undefined } + const changes = destinationAuditDiff({ name: "Ops" }, unknown, view()) + expect(changes).toEqual({ fields: ["name"], before: { name: undefined }, after: { name: "Ops" } }) + }) +}) diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts new file mode 100644 index 000000000..7f781f7ad --- /dev/null +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -0,0 +1,162 @@ +import type { AuditChanges } from "@maple/domain/http" + +/** + * Structural equality, insensitive to object key order (a server-rebuilt + * `timeRange` must not diff against the decoded payload echo). Arrays stay + * order-sensitive; anything non-JSON-shaped falls back to reference equality. + */ +export const structuralEqual = (a: unknown, b: unknown): boolean => { + if (a === b) return true + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false + return a.every((item, index) => structuralEqual(item, b[index])) + } + const aEntries = Object.entries(a) + const bEntries = new Map(Object.entries(b)) + if (aEntries.length !== bEntries.size) return false + return aEntries.every(([key, value]) => bEntries.has(key) && structuralEqual(value, bEntries.get(key))) +} + +/** + * Diff two snapshots restricted to the keys of `after` (the fields the request + * actually touched — omitted fields are unchanged by contract). Returns + * undefined when nothing changed so the audit entry can omit `changes`. + */ +export const diffAuditChanges = ( + before: Record, + after: Record, +): AuditChanges | undefined => { + const fields: string[] = [] + const beforeOut: Record = {} + const afterOut: Record = {} + for (const key of Object.keys(after)) { + const prev = before[key] + const next = after[key] + if (structuralEqual(prev, next)) continue + fields.push(key) + beforeOut[key] = prev + afterOut[key] = next + } + return fields.length === 0 ? undefined : { fields, before: beforeOut, after: afterOut } +} + +/** + * Snapshot only the fields the update payload actually carries, reading their + * values from a wire-shaped view of the resource (pre- or post-update). + */ +export const pickPresentFields = ( + keys: ReadonlyArray, + payload: { readonly [P in K]?: unknown }, + source: { readonly [P in K]: unknown }, +): Record => { + const out: Record = {} + for (const key of keys) { + if (payload[key] !== undefined) out[key] = source[key] + } + return out +} + +/** + * Replace selected fields' before/after values with a static placeholder so + * large config blobs (dashboard widgets, query drafts) and secrets don't reach + * the audit row. Null survives, so "cleared" still reads as cleared. + */ +export const compactAuditChanges = ( + changes: AuditChanges | undefined, + // Keyed by the resource's declared field names (see `auditDiff`) so a wire-key + // rename cannot silently disable a placeholder. + placeholders: Record, +): AuditChanges | undefined => { + if (changes === undefined) return undefined + const before = { ...changes.before } + const after = { ...changes.after } + for (const field of changes.fields) { + const placeholder = placeholders[field] + if (placeholder === undefined) continue + if (field in before && before[field] !== null) before[field] = placeholder + if (field in after && after[field] !== null) after[field] = placeholder + } + return { fields: changes.fields, before, after } +} + +/** + * Strip userinfo, query string, and fragment from a URL destined for an audit + * row — scrape URLs routinely embed tokens there. Keeps scheme/host/path. + */ +export const redactAuditUrl = (raw: string): string => { + if (!URL.canParse(raw)) return "" + const url = new URL(raw) + return `${url.protocol}//${url.host}${url.pathname}` +} + +/** + * Build the `changes` diff for one resource's update handler. + * + * The spec is declared once next to the resource's wire shape and applied per + * request: `fields` are diffed through the wire view, `summarize` replaces a + * config blob's value with a static placeholder, `redact` rewrites a value + * (scrape URLs carry tokens), `writeOnly` records credentials the response + * never echoes as having rotated, and `opaque` records a knob the response does + * not echo either but which is no secret — a channel id, a chat id — as simply + * touched. `summarize` and `redact` are keyed by + * `fields`, so a renamed wire key is a type error rather than a silently + * disabled redaction. + * + * Returns undefined when nothing observable changed, so the caller passes the + * result straight through as `changes`. + */ +const redactedField = (field: string): readonly [string, string] => [field, ""] +const updatedField = (field: string): readonly [string, string] => [field, ""] + +export const auditDiff = (spec: { + readonly fields: ReadonlyArray + readonly summarize?: Partial> + readonly redact?: Partial string>> + readonly writeOnly?: ReadonlyArray + readonly opaque?: ReadonlyArray +}) => { + const redactors: Record string) | undefined> = spec.redact ?? {} + + const redactChanges = (changes: AuditChanges): AuditChanges => { + const apply = (values: Record): Record => { + const out = { ...values } + for (const field of changes.fields) { + const redact = redactors[field] + const value = out[field] + if (redact !== undefined && typeof value === "string") out[field] = redact(value) + } + return out + } + return { fields: changes.fields, before: apply(changes.before), after: apply(changes.after) } + } + + return ( + payload: { readonly [P in Field]?: unknown }, + before: { readonly [P in Field]: unknown }, + after: { readonly [P in Field]: unknown }, + ): AuditChanges | undefined => { + const diffed = diffAuditChanges( + pickPresentFields(spec.fields, payload, before), + pickPresentFields(spec.fields, payload, after), + ) + const compacted = diffed === undefined ? undefined : compactAuditChanges(diffed, spec.summarize ?? {}) + const observable = compacted === undefined ? undefined : redactChanges(compacted) + // Neither kind appears in a response, so that the request carried them is + // the only evidence they changed. They differ in what may be said about + // them: a credential's value is withheld, a channel id's is simply not + // known here. + const present: Record = payload + const touched = [ + ...(spec.writeOnly ?? []).filter((field) => present[field] !== undefined).map(redactedField), + ...(spec.opaque ?? []).filter((field) => present[field] !== undefined).map(updatedField), + ] + if (touched.length === 0) return observable + const placeholders = Object.fromEntries(touched) + return { + fields: [...(observable?.fields ?? []), ...touched.map(([field]) => field)], + before: { ...observable?.before, ...placeholders }, + after: { ...observable?.after, ...placeholders }, + } + } +} diff --git a/apps/api/src/routes/v2/audit-log.http.test.ts b/apps/api/src/routes/v2/audit-log.http.test.ts new file mode 100644 index 000000000..fffde2bf2 --- /dev/null +++ b/apps/api/src/routes/v2/audit-log.http.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "@effect/vitest" +import { Schema } from "effect" +import { UserId } from "@maple/domain/primitives" +import type { AuditLogEntry } from "@/services/audit/audit-event" +import { actorAvatarUrl, actorDisplayName, type ActorProfile } from "./audit-log.http" + +const USER = Schema.decodeUnknownSync(UserId)("user_audit_route_test") + +const directory = (profile: ActorProfile) => new Map([[USER, profile]]) +const ada: ActorProfile = { name: "Ada Lovelace", imageUrl: "https://img.test/ada.png" } + +type Row = Pick + +describe("actorDisplayName", () => { + it("prefers the label frozen at write time over the current directory", () => { + const row: Row = { actorLabel: "Deploy bot", userId: USER, actorType: "api_key" } + expect(actorDisplayName(row, directory(ada))).toBe("Deploy bot") + }) + + it("names a dashboard actor from the directory", () => { + const row: Row = { actorLabel: null, userId: USER, actorType: "user" } + expect(actorDisplayName(row, directory(ada))).toBe("Ada Lovelace") + }) + + // A member who has since left the org is exactly the actor an audit reader + // cares about, so an unresolvable id must still render as itself rather than + // dropping the row or erroring. + it("leaves a departed member unnamed", () => { + const row: Row = { actorLabel: null, userId: USER, actorType: "user" } + expect(actorDisplayName(row, new Map())).toBeNull() + }) + + // An API-key row carries the minting user's id. Naming it from the directory + // would print a person's name on an action a key took. + it("does not lend a minting user's name to their API key", () => { + const row: Row = { actorLabel: null, userId: USER, actorType: "api_key" } + expect(actorDisplayName(row, directory(ada))).toBeNull() + }) + + it("has nothing to name for a system entry", () => { + expect(actorDisplayName({ actorLabel: null, userId: null, actorType: "system" }, new Map())).toBeNull() + }) +}) + +describe("actorAvatarUrl", () => { + it("shows the directory avatar for a user", () => { + expect(actorAvatarUrl({ actorType: "user", userId: USER }, directory(ada))).toBe( + "https://img.test/ada.png", + ) + }) + + // The key acted, not the person who minted it: showing that person's face + // would misattribute the action to a human who may not have been involved. + it("gives an API key no face even though the entry carries a user id", () => { + expect(actorAvatarUrl({ actorType: "api_key", userId: USER }, directory(ada))).toBeNull() + }) + + it("has none for a member the directory does not know", () => { + expect(actorAvatarUrl({ actorType: "user", userId: USER }, new Map())).toBeNull() + }) + + it("tolerates a member with no avatar", () => { + const noFace: ActorProfile = { name: "Ada Lovelace", imageUrl: null } + expect(actorAvatarUrl({ actorType: "user", userId: USER }, directory(noFace))).toBeNull() + }) +}) diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts new file mode 100644 index 000000000..800d39bc2 --- /dev/null +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -0,0 +1,247 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { CurrentTenant } from "@maple/domain/http" +import { ActorId, ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import { + decodePublicId, + encodePublicId, + MapleApiV2, + paginateOffsetQuery, + PublicIdPrefixes, + timestamp, + V2InsufficientPermissions, + V2ParameterInvalid, +} from "@maple/domain/http/v2" +import type { V2AuditLogEntry } from "@maple/domain/http/v2" +import type { AuditLogEntry } from "@/services/audit/audit-event" +import { Cause, Effect, Option, Schema } from "effect" +import { summarizeCause } from "@/platform/describe-cause" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { OrgMembersService } from "@/services/org/OrgMembersService" +import { requireAdmin } from "@/services/auth/auth" +import type { AuditLogListFilters } from "@/services/audit/AuditLogService" + +const adminOnly = () => V2InsufficientPermissions.make("Only org admins can read the audit log") + +/** No directory, no names — the entries still carry every id they were written with. */ +const unnamed = (cause: Cause.Cause) => + Effect.logWarning("Audit log: member directory unavailable; entries keep their ids").pipe( + Effect.annotateLogs({ error: summarizeCause(cause) }), + Effect.as(new Map()), + ) + +const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) +const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) +const decodeUserIdOption = Schema.decodeUnknownOption(UserId) + +type ActorIdentityFilter = Pick + +/** + * Resolve the public `actor_id` filter to the column it identifies: `key_…` → + * the API key, `actor_…` → the agent, anything else → a (Clerk-issued, already + * public) user ID. + */ +const actorIdentityFilter = (publicActorId: string) => { + const invalid = V2ParameterInvalid.make("Invalid actor_id.", { param: "actor_id" }) + const succeed = (filter: ActorIdentityFilter) => Effect.succeed(filter) + const asApiKey = decodePublicId(PublicIdPrefixes.apiKey, publicActorId) + if (asApiKey !== null) { + return Option.match(decodeApiKeyIdOption(asApiKey), { + onNone: () => Effect.fail(invalid), + onSome: (apiKeyId) => succeed({ apiKeyId }), + }) + } + const asActor = decodePublicId(PublicIdPrefixes.actor, publicActorId) + if (asActor !== null) { + return Option.match(decodeActorIdOption(asActor), { + onNone: () => Effect.fail(invalid), + onSome: (actorId) => succeed({ actorId }), + }) + } + return Option.match(decodeUserIdOption(publicActorId), { + onNone: () => Effect.fail(invalid), + onSome: (userId) => succeed({ userId }), + }) +} + +/** The actor's public identifier, matching the ID style of its own resource. */ +const publicActorId = (row: AuditLogEntry): string | null => { + switch (row.actorType) { + case "api_key": + return row.apiKeyId === null ? null : encodePublicId(PublicIdPrefixes.apiKey, row.apiKeyId) + case "agent": + return row.actorId === null ? null : encodePublicId(PublicIdPrefixes.actor, row.actorId) + case "user": + // Clerk user IDs are already prefixed public IDs — passed through as-is. + return row.userId + case "system": + return null + } +} + +/** What the workspace directory can tell us about one member. */ +export interface ActorProfile { + readonly name: string + readonly imageUrl: string | null +} + +/** + * The name to show for one entry: the label frozen at write time when there is + * one, else the current directory name — but only for a user actor, and only + * ever their own. + * + * Every API-key and agent row also carries a `userId`, the person who minted + * the credential. Naming those rows from the directory puts that person's name + * on an action a key took, which is precisely the attribution an audit log + * exists to keep straight. Entries written before keys carried their name show + * an id, which is honest. + */ +export const actorDisplayName = ( + row: Pick, + directory: ReadonlyMap, +): string | null => + row.actorLabel ?? + (row.actorType !== "user" || row.userId === null + ? null + : (directory.get(row.userId)?.name ?? null)) + +/** + * The avatar, for user actors only. An API key or an agent has no face, and a + * departed member has no directory entry to take one from. + */ +export const actorAvatarUrl = ( + row: Pick, + directory: ReadonlyMap, +): string | null => + row.actorType !== "user" || row.userId === null + ? null + : (directory.get(row.userId)?.imageUrl ?? null) + +/** + * Name the humans. An API key freezes its name into `actorLabel` when the entry + * is written, which is what an audit trail wants — the name the credential had + * at the time. A dashboard session has nothing to freeze: Clerk's claims carry + * no name, and resolving one per write would put a directory call on every + * telemetry read. So user rows are labelled here, from the directory as it + * stands, and an id that no longer belongs to a member simply keeps showing as + * an id. + */ +const toV2AuditLogEntry = ( + row: AuditLogEntry, + directory: ReadonlyMap, +): V2AuditLogEntry => ({ + id: row.id, + object: "audit_log_entry", + action: row.action, + outcome: row.outcome, + denial_reason: row.denialReason, + actor_type: row.actorType, + actor_id: publicActorId(row), + actor_name: actorDisplayName(row, directory), + actor_avatar_url: actorAvatarUrl(row, directory), + affected_user: row.affectedUserId, + source: row.source, + resource_type: row.resourceType, + resource_id: row.resourceId, + changes: row.changes, + metadata: row.metadata, + request_id: row.requestId, + origin_ip: row.originIp, + origin_country: row.originCountry, + occurred_at: timestamp(row.occurredAt.toISOString()), + recorded_at: timestamp(row.recordedAt.toISOString()), +}) + +export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", (handlers) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const members = yield* OrgMembersService + + /** + * Names and avatars for the acting users, or nothing at all: a directory + * that is unconfigured (self-hosted without Clerk) or briefly unavailable + * must never turn reading the audit log into an error. Ids still render. + * + * A member with no name set falls back to their email, which is what the + * rest of the product shows and what an admin reading an audit trail + * actually recognises — an opaque `user_…` is the last resort, not the + * second one. + * + * Failures and defects both fall back; interruption is re-raised, because + * a request being torn down has no page left to label. + */ + const directory = (orgId: OrgId) => + members.listMembers(orgId).pipe( + Effect.map( + (all) => + new Map( + all.map( + (member) => + [ + member.userId, + { name: member.name ?? member.email, imageUrl: member.imageUrl }, + ] as const, + ), + ), + ), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) ? Effect.interrupt : unnamed(cause), + ), + ) + + return handlers.handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + // The log carries every member's activity, denial history, and origin + // IP for the whole retention window — org admins only. Scoped API keys + // are additionally gated by `audit_log:read`. + yield* requireAdmin(tenant.roles, adminOnly) + const identity = + query.actor_id !== undefined ? yield* actorIdentityFilter(query.actor_id) : undefined + const affectedUser = + query.affected_user !== undefined + ? yield* Option.match(decodeUserIdOption(query.affected_user), { + onNone: () => + Effect.fail( + V2ParameterInvalid.make("Invalid affected_user.", { param: "affected_user" }), + ), + onSome: (userId) => Effect.succeed(userId), + }) + : undefined + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => + audit + .list(tenant.orgId, { + ...(query.actor_type !== undefined ? { actorType: query.actor_type } : undefined), + ...identity, + ...(affectedUser !== undefined ? { affectedUserId: affectedUser } : undefined), + ...(query.action !== undefined ? { action: query.action } : undefined), + ...(query.outcome !== undefined ? { outcome: query.outcome } : undefined), + ...(query.resource_type !== undefined + ? { resourceType: query.resource_type } + : undefined), + ...(query.resource_id !== undefined + ? { resourceId: query.resource_id } + : undefined), + ...(query.changed !== undefined ? { changedField: query.changed } : undefined), + ...(query.request_id !== undefined ? { requestId: query.request_id } : undefined), + ...(query.since !== undefined ? { sinceMs: Date.parse(query.since) } : undefined), + ...(query.until !== undefined ? { untilMs: Date.parse(query.until) } : undefined), + limit, + offset, + }) + .pipe( + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed([]) + : directory(tenant.orgId).pipe( + Effect.map((known) => + rows.map((row) => toV2AuditLogEntry(row, known)), + ), + ), + ), + ), + ) + return { object: "list" as const, ...page } + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index 19cf64993..1daca7a6c 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -9,6 +9,7 @@ import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQue import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -115,6 +116,7 @@ const makeHarness = () => { // session_replays (in AllV2GroupLayersLive) needs the warehouse at the routes level. Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 025d06de8..2a414c511 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -11,6 +11,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { V2TransportErrorBoundaryLive } from "./error-envelope" import { AlertsServiceStubLayer, @@ -65,6 +66,7 @@ const makeHarness = () => { Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.ts b/apps/api/src/routes/v2/dashboards.http.ts index efc99098a..1a9420477 100644 --- a/apps/api/src/routes/v2/dashboards.http.ts +++ b/apps/api/src/routes/v2/dashboards.http.ts @@ -9,9 +9,11 @@ import { PortableDashboardDocument, } from "@maple/domain/http" import { + encodePublicId, MapleApiV2, LIST_LIMIT_DEFAULT, paginateArray, + PublicIdPrefixes, V2ParameterInvalid, V2ParameterMissing, } from "@maple/domain/http/v2" @@ -30,6 +32,8 @@ import type { DashboardId } from "@maple/domain/primitives" import { Clock, Effect, Option, Schema } from "effect" import { getTemplateById, listTemplateMetadata } from "@/dashboard-templates" import type { TemplateParameterValues } from "@/dashboard-templates" +import { auditDiff } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { convertPersesDashboardToPortable } from "@/services/dashboards/perses-dashboard-import" @@ -174,6 +178,22 @@ const applyUpdate = ( }) } +/** Update-payload fields diffable through the wire shape; layout blobs get summarized. */ +const dashboardAuditDiff = auditDiff({ + fields: [ + "name", + "description", + "tags", + "timeRange", + "widgets", + "sections", + "variables", + "refreshIntervalSeconds", + ], + // Layout arrays are config blobs — audit that they changed, not their bodies. + summarize: { widgets: "", sections: "", variables: "" }, +}) + const encodeVersionCursor = (versionNumber: number): string => `ver_${versionNumber.toString(36)}` const decodeVersionCursor = (cursor: string): number | null => { @@ -270,6 +290,16 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards "maple.share.id": created.id, mode: created.mode, }) + yield* recordHttpAudit("dashboard_share.created", { + resourceId: created.id, + metadata: { + mode: created.mode, + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, context.scope.dashboardId), + ...(context.scope.widgetId === null + ? undefined + : { widget_id: context.scope.widgetId }), + }, + }) return toV2DashboardShare(created) }) @@ -284,6 +314,14 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards ) yield* logShare("dashboard share rotated", context, { "maple.share.id": rotated.id }) + // Security event: rotation invalidates the previous public share token. + yield* recordHttpAudit("dashboard_share.rotated", { + resourceId: rotated.id, + metadata: { + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), + ...(widgetId === null ? undefined : { widget_id: widgetId }), + }, + }) return toV2DashboardShare(rotated) }) @@ -298,6 +336,14 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards ) yield* logShare("dashboard share revoked", context, { hadLiveShare: tombstone.revoked }) + if (tombstone.revoked) { + yield* recordHttpAudit("dashboard_share.deleted", { + metadata: { + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), + ...(widgetId === null ? undefined : { widget_id: widgetId }), + }, + }) + } // `deleted: true` regardless of whether a live share existed: "stop // sharing" is a statement about the end state, and the dialog must be @@ -336,6 +382,10 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards tenant.userId, toPortable(payload), ) + yield* recordHttpAudit("dashboard.created", { + resourceId: dashboard.id, + metadata: { name: dashboard.name }, + }) return toV2DashboardMutation(dashboard) }), @@ -346,12 +396,26 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards const updatedAt = asIsoDateTime( new Date(yield* Clock.currentTimeMillis).toISOString(), ) + // Capture the pre-state the mutate callback already reads, for the diff. + let previous: DashboardDocument | undefined const dashboard = yield* persistence.mutate( tenant.orgId, tenant.userId, params.id, - (current) => Effect.succeed(applyUpdate(current, payload, updatedAt)), + (current) => { + previous = current + return Effect.succeed(applyUpdate(current, payload, updatedAt)) + }, ) + const changes = + previous === undefined + ? undefined + : dashboardAuditDiff(payload, toV2Dashboard(previous), toV2Dashboard(dashboard)) + yield* recordHttpAudit("dashboard.updated", { + resourceId: dashboard.id, + changes, + metadata: { name: dashboard.name }, + }) return toV2DashboardMutation(dashboard) }), @@ -360,6 +424,9 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* persistence.delete(tenant.orgId, params.id) + yield* recordHttpAudit("dashboard.deleted", { + resourceId: deleted.id, + }) return { id: deleted.id, @@ -378,6 +445,10 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards tenant.userId, converted.dashboard, ) + yield* recordHttpAudit("dashboard.created", { + resourceId: dashboard.id, + metadata: { name: dashboard.name, source: "perses_import" }, + }) return { object: "dashboard_import" as const, @@ -436,6 +507,16 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards params.id, params.version_id, ) + yield* recordHttpAudit("dashboard.version_restored", { + resourceId: dashboard.id, + metadata: { + name: dashboard.name, + version_id: encodePublicId( + PublicIdPrefixes.dashboardVersion, + params.version_id, + ), + }, + }) return toV2DashboardMutation(dashboard) }), @@ -528,6 +609,14 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards }) const tenant = yield* CurrentTenant.Context const dashboard = yield* persistence.create(tenant.orgId, tenant.userId, portable) + yield* recordHttpAudit("dashboard.created", { + resourceId: dashboard.id, + metadata: { + name: dashboard.name, + source: "template", + template_id: params.template_id, + }, + }) return toV2DashboardMutation(dashboard) }), diff --git a/apps/api/src/routes/v2/ingest-keys.http.ts b/apps/api/src/routes/v2/ingest-keys.http.ts index cddf23be2..28fa08c31 100644 --- a/apps/api/src/routes/v2/ingest-keys.http.ts +++ b/apps/api/src/routes/v2/ingest-keys.http.ts @@ -4,6 +4,7 @@ import { CurrentTenant } from "@maple/domain/http" import { MapleApiV2, V2InsufficientPermissions } from "@maple/domain/http/v2" import type { V2IngestKeys } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" import { requireAdmin } from "@/services/auth/auth" @@ -37,6 +38,9 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPublic(tenant.orgId, tenant.userId) + yield* recordHttpAudit("ingest_key.rolled", { + metadata: { key_type: "public" }, + }) return toV2IngestKeys(keys) }), @@ -46,6 +50,9 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPrivate(tenant.orgId, tenant.userId) + yield* recordHttpAudit("ingest_key.rolled", { + metadata: { key_type: "private" }, + }) return toV2IngestKeys(keys) }), diff --git a/apps/api/src/routes/v2/integrations.http.test.ts b/apps/api/src/routes/v2/integrations.http.test.ts index a21d9b9b0..e9d04f3a3 100644 --- a/apps/api/src/routes/v2/integrations.http.test.ts +++ b/apps/api/src/routes/v2/integrations.http.test.ts @@ -23,6 +23,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { SLACK_CALLBACK_PATH, SlackIntegrationService, @@ -170,6 +171,7 @@ const makeHarness = (slack: Partial = {}, planetscal Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/integrations.http.ts b/apps/api/src/routes/v2/integrations.http.ts index d96912744..4dfbc1c1f 100644 --- a/apps/api/src/routes/v2/integrations.http.ts +++ b/apps/api/src/routes/v2/integrations.http.ts @@ -31,6 +31,7 @@ import { V2TimeRangeInvalid, } from "@maple/domain/http/v2" import { Array as Arr, Effect, Option } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" import { Env } from "@/platform/Env" import { EdgeCacheService } from "@maple/cache" @@ -259,6 +260,7 @@ export const HttpV2SlackIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "sla const result = yield* slack .startInstall(tenant.orgId, tenant.userId, callbackUrl) .pipe(tapHttpErrors("Slack install failed")) + yield* recordHttpAudit("slack_integration.install_started") return { object: "slack_integration.install" as const, url: result.url, @@ -274,6 +276,7 @@ export const HttpV2SlackIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "sla yield* slack .uninstall(tenant.orgId) .pipe(tapHttpErrors("Slack integration uninstall failed")) + yield* recordHttpAudit("slack_integration.uninstalled") return { object: "slack_integration" as const, installed: false as const, @@ -354,6 +357,7 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( returnTo: payload.return_to, }) .pipe(tapHttpErrors("PlanetScale connect failed")) + yield* recordHttpAudit("planetscale_integration.connect_started") return { object: "planetscale_integration.connect" as const, redirect_url: result.redirectUrl, @@ -398,6 +402,13 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( excludeBranches: payload.exclude_branches, }) .pipe(tapHttpErrors("PlanetScale organization selection failed")) + yield* recordHttpAudit("planetscale_integration.organization_selected", { + metadata: { + organization: payload.organization, + include_branches: payload.include_branches, + exclude_branches: payload.exclude_branches, + }, + }) return toPlanetScaleStatus(status) }), ) @@ -415,6 +426,11 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( tokenSecret: payload.token_secret, }) .pipe(tapHttpErrors("PlanetScale metrics token update failed")) + // The token id names which credential was installed; its secret + // is write-only and never reaches an audit row. + yield* recordHttpAudit("planetscale_integration.metrics_token_set", { + metadata: { token_id: payload.token_id }, + }) return toPlanetScaleStatus(status) }), ) @@ -427,6 +443,7 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( yield* planetscale .disconnect(tenant.orgId) .pipe(tapHttpErrors("PlanetScale disconnect failed")) + yield* recordHttpAudit("planetscale_integration.disconnected") return { object: "planetscale_integration" as const, connected: false as const, diff --git a/apps/api/src/routes/v2/investigations.http.ts b/apps/api/src/routes/v2/investigations.http.ts index 54388f0d7..31fed9372 100644 --- a/apps/api/src/routes/v2/investigations.http.ts +++ b/apps/api/src/routes/v2/investigations.http.ts @@ -21,6 +21,7 @@ import type { V2InvestigationSubject, } from "@maple/domain/http/v2" import { Effect, Match, Schema } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { InvestigationService } from "@/services/errors/InvestigationService" const toWireSubject = Effect.fn("HttpV2Investigations.toWireSubject")(function* ( @@ -265,6 +266,10 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest : undefined), }), ) + yield* recordHttpAudit("investigation.created", { + resourceId: doc.id, + metadata: { subject_type: payload.subject.type }, + }) return yield* serializeInvestigation(doc) }), @@ -273,6 +278,7 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const doc = yield* service.restartInvestigation(tenant.orgId, params.id) + yield* recordHttpAudit("investigation.restarted", { resourceId: doc.id }) return yield* serializeInvestigation(doc) }), @@ -281,6 +287,10 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const doc = yield* service.updateStatus(tenant.orgId, params.id, payload.status) + yield* recordHttpAudit("investigation.status_changed", { + resourceId: doc.id, + metadata: { to_status: payload.status }, + }) return yield* serializeInvestigation(doc) }), diff --git a/apps/api/src/routes/v2/mobile-devices.http.test.ts b/apps/api/src/routes/v2/mobile-devices.http.test.ts index 416e0826a..86b2c14a1 100644 --- a/apps/api/src/routes/v2/mobile-devices.http.test.ts +++ b/apps/api/src/routes/v2/mobile-devices.http.test.ts @@ -7,6 +7,7 @@ import { MapleApiV2, encodePublicId } from "@maple/domain/http/v2" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -79,6 +80,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index 697ea380d..12771c1bb 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -50,6 +50,7 @@ import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryServic import { Env } from "@/platform/Env" import { AnomalyDetectionService } from "@/services/alerts/AnomalyDetectionService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -562,6 +563,7 @@ const makeHarness = ( Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/scrape-targets.http.ts b/apps/api/src/routes/v2/scrape-targets.http.ts index 83f15b09e..0a99043db 100644 --- a/apps/api/src/routes/v2/scrape-targets.http.ts +++ b/apps/api/src/routes/v2/scrape-targets.http.ts @@ -10,6 +10,8 @@ import { } from "@maple/domain/http/v2" import type { V2ScrapeTarget, V2ScrapeTargetCheck } from "@maple/domain/http/v2" import { Effect } from "effect" +import { auditDiff, redactAuditUrl } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ScrapeTargetsService } from "@/services/integrations/ScrapeTargetsService" import { requireAdmin } from "@/services/auth/auth" @@ -42,6 +44,27 @@ const toV2ScrapeTarget = (target: ScrapeTargetResponse): V2ScrapeTarget => ({ updated_at: target.updatedAt, }) +/** Update-payload fields diffable through the wire shape; credentials never appear. */ +const targetAuditDiff = auditDiff({ + fields: [ + "name", + "url", + "organization", + "include_branches", + "exclude_branches", + "scrape_interval_seconds", + "labels_json", + "auth_type", + "service_name", + "enabled", + ], + // Scrape URLs may carry tokens in userinfo/query — audit only scheme/host/path. + // Identical redacted values still mean the URL changed within the stripped part. + redact: { url: redactAuditUrl }, + // Credentials are write-only: audit that they rotated, never their value. + writeOnly: ["auth_credentials"], +}) + export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeTargets", (handlers) => Effect.gen(function* () { const service = yield* ScrapeTargetsService @@ -111,6 +134,11 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT }), ) + yield* recordHttpAudit("scrape_target.created", { + resourceId: created.id, + metadata: { name: created.name }, + }) + return toV2ScrapeTarget(created) }), ) @@ -118,6 +146,7 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("update")) + const current = yield* service.get(tenant.orgId, params.id) const updated = yield* service.update( tenant.orgId, params.id, @@ -160,6 +189,14 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT }), ) + // Read-then-write with no CAS: a concurrent update can make `before` + // reflect a state this update never saw. Accepted for audit purposes. + yield* recordHttpAudit("scrape_target.updated", { + resourceId: updated.id, + changes: targetAuditDiff(payload, toV2ScrapeTarget(current), toV2ScrapeTarget(updated)), + metadata: { name: updated.name }, + }) + return toV2ScrapeTarget(updated) }), ) @@ -168,6 +205,7 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("delete")) const deleted = yield* service.delete(tenant.orgId, params.id) + yield* recordHttpAudit("scrape_target.deleted", { resourceId: deleted.id }) return { id: deleted.id, object: "scrape_target" as const, deleted: true as const } }), diff --git a/apps/api/src/routes/v2/setup-audit.http.test.ts b/apps/api/src/routes/v2/setup-audit.http.test.ts index 79698faa6..e3c6a7247 100644 --- a/apps/api/src/routes/v2/setup-audit.http.test.ts +++ b/apps/api/src/routes/v2/setup-audit.http.test.ts @@ -10,6 +10,7 @@ import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryServic import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -142,6 +143,7 @@ const makeHarness = (warehouse: WarehouseQueryServiceApi = warehouseStub()) => { Layer.provide(TelemetryServiceStubsLayer), Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index 213e1dcba..c59e59eaf 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -12,6 +12,7 @@ import { type WarehouseQueryServiceApi, } from "@/services/warehouse/WarehouseQueryService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -275,6 +276,7 @@ const makeHarness = ( Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 6e1342988..27c3fd826 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -42,6 +42,9 @@ import { HttpV2InvestigationsLive } from "./investigations.http" import { HttpV2MobileDevicesLive } from "./mobile-devices.http" import { HttpV2OrganizationLive } from "./organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./recommendations.http" +import { HttpV2AuditLogLive } from "./audit-log.http" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { OrgMembersService } from "@/services/org/OrgMembersService" import { HttpV2ScrapeTargetsLive } from "./scrape-targets.http" import { HttpV2SessionReplaysLive } from "./session-replays.http" import { HttpV2InstrumentationAuditLive } from "./setup-audit.http" @@ -65,6 +68,15 @@ import { HttpV2WidgetCredentialsLive } from "./widget-credentials.http" * the groups it does not exercise. */ +/** + * An empty workspace directory: the audit log falls back to rendering ids, + * which is the same shape a self-hosted deployment without Clerk sees. + */ +export const OrgMembersServiceStubLayer = Layer.succeed(OrgMembersService, { + listMembers: () => Effect.succeed([]), + resolveMembers: () => Effect.succeed([]), +}) + export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2ApiKeysLive, HttpV2SlackIntegrationsLive, @@ -77,6 +89,12 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2IngestKeysLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, + // Real service, no stub: it needs only the Database every harness already + // provides. The member directory IS stubbed — the route asks it for display + // names, and every harness would otherwise reach for Clerk. + HttpV2AuditLogLive.pipe( + Layer.provide(Layer.merge(AuditLogService.layerMemory, OrgMembersServiceStubLayer)), + ), HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, @@ -119,6 +137,10 @@ export const AllV2GroupLayersLive = Layer.mergeAll( }), ), ), +).pipe( + // Mutation handlers across the groups record audit entries; the real service + // needs only the Database every harness already provides. + Layer.provide(AuditLogService.layerMemory), ) export const ApiV2RateLimiterAllowAllLayer = Layer.succeed(ApiV2RateLimiter, { diff --git a/apps/api/src/routes/v2/widget-credentials.http.test.ts b/apps/api/src/routes/v2/widget-credentials.http.test.ts index 05b784def..b785222b2 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.test.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.test.ts @@ -7,6 +7,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -77,6 +78,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/widget-credentials.http.ts b/apps/api/src/routes/v2/widget-credentials.http.ts index 3ae43c5d6..15e57c8b2 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.ts @@ -2,6 +2,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant } from "@maple/domain/http" import { MapleApiV2, isoTimestamp } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" /** @@ -49,6 +50,14 @@ export const HttpV2WidgetCredentialsLive = HttpApiBuilder.group(MapleApiV2, "wid // letting it resolve with the API-key default — is `root`. roles: tenant.roles, }) + // A credential mint is the security event; the secret itself never + // reaches the row, only which installation it was issued to. + yield* recordHttpAudit("widget_credential.minted", { + metadata: { + installation_id: params.installation_id, + scopes: credential.scopes ?? WIDGET_CREDENTIAL_SCOPES, + }, + }) return { object: "widget_credential" as const, secret: credential.secret, @@ -68,6 +77,9 @@ export const HttpV2WidgetCredentialsLive = HttpApiBuilder.group(MapleApiV2, "wid // to revoke: this is the sign-out path, and an error the app // cannot act on while signing out anyway is worse than silence. yield* apiKeys.revokeDeviceKeys(tenant.orgId, params.installation_id) + yield* recordHttpAudit("widget_credential.revoked", { + metadata: { installation_id: params.installation_id }, + }) return { object: "widget_credential" as const, deleted: true as const } }), ) diff --git a/apps/api/src/routes/v2/widget-summary.http.test.ts b/apps/api/src/routes/v2/widget-summary.http.test.ts index 6113aeb70..fc25fb99b 100644 --- a/apps/api/src/routes/v2/widget-summary.http.test.ts +++ b/apps/api/src/routes/v2/widget-summary.http.test.ts @@ -15,6 +15,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -190,6 +191,7 @@ const makeHarness = (options: { Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/webhooks/clerk.http.ts b/apps/api/src/routes/webhooks/clerk.http.ts index 9e79573a7..51c863f32 100644 --- a/apps/api/src/routes/webhooks/clerk.http.ts +++ b/apps/api/src/routes/webhooks/clerk.http.ts @@ -4,21 +4,30 @@ import { HttpRouter, type HttpServerRequest } from "effect/unstable/http" import { Env } from "@/platform/Env" import { MembershipRevocationService } from "@/services/auth/MembershipRevocationService" import { + CLERK_MEMBERSHIP_EVENTS, decodeClerkEnvelope, decodeClerkOrganizationMembership, decodeClerkUserCreated, decodeClerkUserDeleted, + isClerkMembershipEvent, signupCompletedEvent, + type ClerkMembershipEventType, } from "@/services/product-events/clerk-events" import { ProductEventsService } from "@/services/product-events/ProductEventsService" +import { AuditLogService } from "@/services/audit/AuditLogService" import { receiveSvixWebhook, webhookText } from "./svix-receiver" /** - * Clerk webhook receiver. Two jobs: + * Clerk webhook receiver. Three jobs: * * - `user.created` → `signup_completed` product event. - * - membership lifecycle → the revocation sweep. This is the only thing in - * Maple that runs when somebody stops being a member of an organization; + * - membership lifecycle → an org audit entry (`member.added` / `role_changed` + * / `removed`). Membership is the one org change the web app makes in Clerk + * rather than through Maple's API, so this receiver is the only writer of + * `affected_user`. Enabling the three `organizationMembership.*` events in + * the Clerk dashboard is what turns it on. + * - membership removal/demotion → the revocation sweep. This is the only thing + * in Maple that runs when somebody stops being a member of an organization; * without it the membership cache and every user-bound credential keep * answering yes indefinitely. * @@ -35,6 +44,7 @@ const handler = Effect.gen(function* () { const env = yield* Env const productEvents = yield* ProductEventsService const revocation = yield* MembershipRevocationService + const audit = yield* AuditLogService /** * A revocation that only half-ran is the state this whole handler exists to @@ -62,7 +72,7 @@ const handler = Effect.gen(function* () { const handleMembership = Effect.fn("ClerkWebhook.membership")(function* ( data: unknown, - removed: boolean, + event: ClerkMembershipEventType, ) { const payload = yield* decodeClerkOrganizationMembership(data).pipe(Effect.option) if (Option.isNone(payload)) { @@ -81,7 +91,28 @@ const handler = Effect.gen(function* () { }) return webhookText("Unrecognized membership identifiers", 400) } - if (removed) return yield* revoke(revocation.revokeMembership(orgId.value, userId.value)) + // Clerk's payload names the member, never the admin who acted, so + // attributing this to a user would be a guess. `system` says truthfully + // that Maple learned of the change rather than made it. Recorded before + // the sweep: a revocation that fails and retries must not lose the entry. + yield* audit.record({ + orgId: orgId.value, + actor: { type: "system" }, + source: "system", + action: `member.${CLERK_MEMBERSHIP_EVENTS[event]}`, + affectedUserId: userId.value, + ...(payload.value.role !== undefined ? { metadata: { role: payload.value.role } } : undefined), + }) + if (event === "organizationMembership.created") { + yield* Effect.annotateCurrentSpan({ + "http.response.status_code": 200, + "maple.webhook.outcome": "handled", + }) + return webhookText("ok", 200) + } + if (event === "organizationMembership.deleted") { + return yield* revoke(revocation.revokeMembership(orgId.value, userId.value)) + } // A role change is not just a cache eviction: CLI/MCP/device keys pin the // minting user's roles and are never re-checked, so a demotion has to go @@ -121,11 +152,8 @@ const handler = Effect.gen(function* () { } yield* Effect.annotateCurrentSpan({ "maple.webhook.event": envelope.value.type }) - if (envelope.value.type === "organizationMembership.deleted") { - return yield* handleMembership(envelope.value.data, true) - } - if (envelope.value.type === "organizationMembership.updated") { - return yield* handleMembership(envelope.value.data, false) + if (isClerkMembershipEvent(envelope.value.type)) { + return yield* handleMembership(envelope.value.data, envelope.value.type) } if (envelope.value.type === "user.deleted") { const payload = yield* decodeClerkUserDeleted(envelope.value.data).pipe(Effect.option) diff --git a/apps/api/src/routes/webhooks/webhooks.http.test.ts b/apps/api/src/routes/webhooks/webhooks.http.test.ts index 5934d8e71..1af291049 100644 --- a/apps/api/src/routes/webhooks/webhooks.http.test.ts +++ b/apps/api/src/routes/webhooks/webhooks.http.test.ts @@ -4,6 +4,7 @@ import { HttpRouter } from "effect/unstable/http" import { Env } from "@/platform/Env" import { ProductEventsService, type ProductEventInput } from "@/services/product-events/ProductEventsService" import { signSvix } from "@/services/product-events/svix" +import { AuditLogService, type AuditLogRecordInput } from "@/services/audit/AuditLogService" import { MembershipRevocationService, MembershipRevocationError, @@ -39,6 +40,15 @@ const recordingProductEvents = () => { return { tracked, layer } } +const recordingAudit = () => { + const recorded: Array = [] + const layer = Layer.succeed(AuditLogService, { + record: (input) => Effect.sync(() => void recorded.push(input)), + list: () => Effect.succeed([]), + }) + return { recorded, layer } +} + const EMPTY_SUMMARY: MembershipRevocationSummary = { apiKeysRevoked: 0, mcpFamiliesRevoked: 0, @@ -80,10 +90,12 @@ const makeRouterLayer = ( config: Record, productEvents: Layer.Layer, revocation: Layer.Layer = recordingRevocation().layer, + audit: Layer.Layer = recordingAudit().layer, ) => router.pipe( Layer.provide(productEvents), Layer.provide(revocation), + Layer.provide(audit), Layer.provide(Env.layer), Layer.provide(makeConfig(config)), ) @@ -166,7 +178,56 @@ const AUTUMN_BILLING_UPDATED = JSON.stringify({ }, }) +const CLERK_MEMBERSHIP_CREATED = JSON.stringify({ + type: "organizationMembership.created", + timestamp: 1_700_000_000_000, + data: { + organization: { id: "org_42" }, + public_user_data: { user_id: "user_2abc" }, + role: "org:admin", + }, +}) + describe("ClerkWebhookRouter", () => { + // Membership is changed in Clerk, never through Maple's API, so this receiver + // is the only writer of `affected_user`. + it.effect("audits an organizationMembership.created delivery against the member", () => + Effect.gen(function* () { + const events = recordingProductEvents() + const audit = recordingAudit() + const configured = HttpRouter.toWebHandler( + makeRouterLayer( + ClerkWebhookRoute, + { CLERK_WEBHOOK_SECRET: CLERK_SECRET }, + events.layer, + recordingRevocation().layer, + audit.layer, + ), + { disableLogger: true }, + ) + yield* Effect.gen(function* () { + const now = Date.now() + const headers = yield* signedHeaders(CLERK_SECRET, CLERK_MEMBERSHIP_CREATED, now) + const response = yield* post( + configured.handler, + "/webhooks/clerk", + CLERK_MEMBERSHIP_CREATED, + headers, + ) + assert.strictEqual(response.status, 200) + assert.strictEqual(audit.recorded.length, 1) + const entry = audit.recorded[0]! + assert.strictEqual(entry.action, "member.added") + assert.strictEqual(entry.affectedUserId, "user_2abc") + assert.strictEqual(entry.orgId, "org_42") + // Clerk's payload never names the admin who acted; claiming a user + // here would be a guess, so the entry is Maple recording what it learned. + assert.strictEqual(entry.actor.type, "system") + assert.strictEqual(entry.source, "system") + }).pipe(Effect.ensuring(Effect.promise(() => configured.dispose()))) + }), + ) + it.effect( "503s while unconfigured, 401s a bad signature, and emits signup_completed for user.created", () => diff --git a/apps/api/src/runtime/graph-boundaries.test.ts b/apps/api/src/runtime/graph-boundaries.test.ts index cadd8f9eb..03ee0b1b2 100644 --- a/apps/api/src/runtime/graph-boundaries.test.ts +++ b/apps/api/src/runtime/graph-boundaries.test.ts @@ -71,6 +71,8 @@ describe("API runtime graph boundaries", () => { "AlertReadModelsServiceLive", "AlertRulesServiceLive", "AlertsServiceLive", + // Lets `register_agent` (and issue-workflow mutations) write org audit entries. + "AuditLogServiceLive", "DashboardPersistenceService.layer", "ErrorActorsServiceLive", "ErrorIssueReadModelsServiceLive", diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 6177b5629..595e9f771 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -48,6 +48,8 @@ import { HttpV2InvestigationsLive } from "@/routes/v2/investigations.http" import { HttpV2MobileDevicesLive } from "@/routes/v2/mobile-devices.http" import { HttpV2OrganizationLive } from "@/routes/v2/organization.http" import { HttpV2InstrumentationRecommendationsLive } from "@/routes/v2/recommendations.http" +import { HttpV2AuditLogLive } from "@/routes/v2/audit-log.http" +import { AuditLogServiceLive } from "@/runtime/service-graph" import { HttpV2ScrapeTargetsLive } from "@/routes/v2/scrape-targets.http" import { HttpV2InstrumentationAuditLive } from "@/routes/v2/setup-audit.http" import { HttpV2SessionReplaysLive } from "@/routes/v2/session-replays.http" @@ -130,6 +132,7 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2PlanetScaleIntegrationsLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, + HttpV2AuditLogLive, HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, @@ -184,6 +187,8 @@ export const ApiAuthLive = Layer.mergeAll( Layer.provideMerge(ApiV2RateLimiter.layer), Layer.provideMerge(McpToolRateLimiter.layer), Layer.provideMerge(ApiKeysService.layer), + // Denied attempts and audited reads are recorded from inside the auth layers. + Layer.provideMerge(AuditLogServiceLive), // Membership verification for `x-maple-org-id`. Only the v2 layer asks for // it; without it that layer cannot build, which is deliberate — the header // must never end up silently ignored in a runtime that forgot to wire this. diff --git a/apps/api/src/runtime/mcp-service-graph.ts b/apps/api/src/runtime/mcp-service-graph.ts index 3e89f667f..a23a3234e 100644 --- a/apps/api/src/runtime/mcp-service-graph.ts +++ b/apps/api/src/runtime/mcp-service-graph.ts @@ -2,6 +2,7 @@ import { EdgeCacheService } from "@maple/cache" import { BucketCacheService } from "@maple/query-engine/caching" import { Layer } from "effect" import { McpToolExecutor } from "@/mcp/dispatcher" +import { AuditLogService } from "@/services/audit/AuditLogService" import { CacheBackendLive } from "@/platform/CacheBackendLive" import { EmailService } from "@/platform/EmailService" import { Env } from "@/platform/Env" @@ -50,6 +51,8 @@ const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( Layer.provide(Layer.mergeAll(InfraLive, OrgClickHouseSettingsServiceLive, TinybirdOrgTokenServiceLive)), ) +const AuditLogServiceLive = AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)) + const BucketCacheServiceLive = BucketCacheService.layer.pipe(Layer.provideMerge(EdgeCacheServiceLive)) const QueryEngineServiceLive = QueryEngineService.layer.pipe( @@ -104,7 +107,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(ErrorActorsServiceLive), + Layer.provide(Layer.mergeAll(ErrorActorsServiceLive, AuditLogServiceLive)), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer const ErrorIssueReadModelsServiceLive = ErrorIssueReadModelsService.layer.pipe( @@ -164,6 +167,7 @@ const McpRuntimeServicesLive = Layer.mergeAll( AlertReadModelsServiceLive, AlertRulesServiceLive, AlertsServiceLive, + AuditLogServiceLive, DashboardPersistenceService.layer, ErrorActorsServiceLive, ErrorIssueReadModelsServiceLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 4d584c333..d7f8a34c3 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -53,6 +53,7 @@ import { GithubConnectService } from "@/services/integrations/vcs/vendor/github/ import { GithubHttp } from "@/services/integrations/vcs/vendor/github/GithubHttp" import { GithubProvider } from "@/services/integrations/vcs/vendor/github/GithubProvider" import { ApiKeysService } from "@/services/org/ApiKeysService" +import { AuditLogService } from "@/services/audit/AuditLogService" import { DemoService } from "@/services/org/DemoService" import { IngestAttributeMappingService } from "@/services/org/IngestAttributeMappingService" import { OnboardingService } from "@/services/org/OnboardingService" @@ -110,6 +111,13 @@ const CoreServicesLive = Layer.mergeAll( const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe(Layer.provideMerge(CoreServicesLive)) +/** + * Audit entries are warehouse rows (Tinybird-pinned `ingest`), so the service + * composes after the warehouse rather than inside CoreServicesLive. Exported + * for the auth layers in `http-graph.ts`, which record denials and reads. + */ +export const AuditLogServiceLive = AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)) + // Serves the integration page's per-zone collection status; the poll loop itself // runs in the alerting worker's cron, not here. const CloudflareAnalyticsServiceLive = CloudflareAnalyticsService.layer.pipe( @@ -176,6 +184,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogServiceLive), Layer.provideMerge(ErrorActorsServiceLive), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer @@ -298,6 +307,7 @@ const MainServicesLive = Layer.mergeAll( ProductEventsServiceLive, DailySpendServiceLive, CloudflareAnalyticsServiceLive, + AuditLogServiceLive, WarehouseQueryServiceLive, EdgeCacheServiceLive, QueryEngineServiceLive, diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index e2527bb50..887bb3444 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,4 +1,4 @@ -import { formatWarehouseDateTime, snapAlertWindowEndMs } from "@maple/query-engine" +import { formatWarehouseDateTime, snapAlertWindowEndMs, warehouseDateTime64 } from "@maple/query-engine" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -283,13 +283,7 @@ export const interleaveAlertRulesByOrg = ( return fair } -// Tinybird DateTime64(3) wire format for alert_checks ingest: -// "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). -const toIngestDateTime64 = (epochMs: number) => { - const d = new Date(epochMs) - const pad = (n: number, w = 2) => n.toString().padStart(w, "0") - return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` -} +const toIngestDateTime64 = warehouseDateTime64 const compareThreshold = ( value: number, diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts new file mode 100644 index 000000000..808f64e6f --- /dev/null +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -0,0 +1,404 @@ +import { describe, expect, it } from "@effect/vitest" +import { WorkerEnvironment } from "@maple/infra/worker-runtime" +import { CurrentTenant } from "@maple/domain/http" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" +import { ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditLogRow } from "@maple/domain/tinybird" +import { Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { makeWarehouseServiceStub } from "@/routes/v2/v2-test-support" +import { type AuditActorInfo, CurrentAuditActor } from "@/services/auth/audit-actor" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { AUDIT_LOG_DATASOURCE, AuditLogService, recordHttpAudit } from "./AuditLogService" + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) + +const ORG = asOrgId("org_audit_log_test") +const USER = asUserId("user_audit_log_test") +const DASHBOARD_ID = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" +const API_KEY = Schema.decodeUnknownSync(ApiKeyId)("7b2e4c10-55aa-4d3e-9f21-1a2b3c4d5e6f") + +/** + * A warehouse that records what `ingest` receives and answers `compiledQuery` + * with canned rows, exposing the SQL it was handed so a test can assert which + * filters the listing bound. + */ +const recordingWarehouse = (rows: ReadonlyArray> = []) => { + const ingested: Array<{ datasource: string; rows: ReadonlyArray }> = [] + const sql: Array = [] + const layer = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: (_tenant, datasource, batch) => + Effect.sync(() => { + // SAFETY: this stub only ever receives the audit datasource's rows. + ingested.push({ datasource, rows: batch as ReadonlyArray }) + }), + compiledQuery: ((_tenant: unknown, compiled: unknown) => + Effect.gen(function* () { + const query = Effect.isEffect(compiled) ? yield* compiled : compiled + // SAFETY: every compiled query carries its SQL text. + sql.push((query as { readonly sql: string }).sql) + return rows + })) as never, + }), + ) + return { ingested, sql, layer } +} + +const storedRow = (overrides: Partial> = {}) => ({ + id: "9d2c1e3a-6a1b-4f0e-9c1d-2b3a4c5d6e7f", + occurredAt: "2026-08-29 09:12:00.412", + recordedAt: "2026-08-29 09:12:00.900", + actorType: "user", + userId: USER, + apiKeyId: "", + actorId: "", + actorLabel: "David", + affectedUserId: "", + source: "dashboard", + action: "dashboard.updated", + outcome: "allowed", + denialReason: "", + resourceType: "dashboard", + resourceId: "dash_1", + changedFields: ["name"], + changes: JSON.stringify({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }), + metadata: JSON.stringify({ reason: "rename" }), + requestId: "ray", + originIp: "203.0.113.7", + originCountry: "DE", + ...overrides, +}) + +describe("AuditLogService (warehouse-backed)", () => { + it.effect("writes one audit_log row through ingest, with '' for absent values", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + // Internal ID in, public `dash_…` ID out — the service owns the encoding. + resourceId: DASHBOARD_ID, + metadata: { name: "First" }, + }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + expect(warehouse.ingested).toHaveLength(1) + expect(warehouse.ingested[0]!.datasource).toBe(AUDIT_LOG_DATASOURCE) + const row = warehouse.ingested[0]!.rows[0]! + expect(row.OrgId).toBe(ORG) + expect(row.ActorType).toBe("user") + expect(row.UserId).toBe(USER) + expect(row.ApiKeyId).toBe("") + expect(row.ResourceType).toBe("dashboard") + expect(row.ResourceId).toBe(encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID)) + expect(row.ChangedFields).toEqual([]) + expect(row.Changes).toBe("") + expect(JSON.parse(row.Metadata)).toEqual({ name: "First" }) + expect(row.OccurredAt).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/) + }), + ) + + it.effect("publishes to the audit queue instead of writing when the binding is present", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + const sent: unknown[] = [] + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + }) + }).pipe( + Effect.provide( + AuditLogService.layer.pipe( + Layer.provide(warehouse.layer), + Layer.provide( + Layer.succeed(WorkerEnvironment, { + AUDIT_EVENTS_QUEUE: { + send: async (message: unknown) => { + sent.push(message) + }, + }, + }), + ), + ), + ), + ) + expect(sent).toHaveLength(1) + expect(sent[0]).toMatchObject({ orgId: ORG, action: "dashboard.created" }) + // The consumer performs the write; nothing reaches the warehouse directly. + expect(warehouse.ingested).toEqual([]) + }), + ) + + it.effect("drops the entry rather than writing to the warehouse on the response path", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "api_key" }, + source: "api", + action: "alert_rule.updated", + }) + }).pipe( + Effect.provide( + AuditLogService.layer.pipe( + Layer.provide(warehouse.layer), + Layer.provide( + Layer.succeed(WorkerEnvironment, { + AUDIT_EVENTS_QUEUE: { + send: async () => { + throw new Error("broker down") + }, + }, + }), + ), + ), + ), + ) + // The queue owns durability (retries, DLQ). A failed send must not buy a + // second network round trip with the caller's response time, which is + // exactly when the platform is already degraded — the caller still + // succeeds, and the loss is in the logs. + expect(warehouse.ingested).toHaveLength(0) + }), + ) + + it.effect("decodes stored rows: '' becomes null, documents parse, timestamps are UTC", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse([storedRow()]) + const rows = yield* Effect.gen(function* () { + const audit = yield* AuditLogService + return yield* audit.list(ORG, { limit: 10, offset: 0 }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + expect(rows).toHaveLength(1) + const entry = rows[0]! + expect(entry.orgId).toBe(ORG) + expect(entry.userId).toBe(USER) + expect(entry.apiKeyId).toBeNull() + expect(entry.affectedUserId).toBeNull() + expect(entry.denialReason).toBeNull() + expect(entry.changedFields).toEqual(["name"]) + expect(entry.changes).toEqual({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }) + expect(entry.metadata).toEqual({ reason: "rename" }) + expect(entry.occurredAt.toISOString()).toBe("2026-08-29T09:12:00.412Z") + expect(entry.recordedAt.toISOString()).toBe("2026-08-29T09:12:00.900Z") + }), + ) + + it.effect("an entry without a diff lists with null changed fields", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse([storedRow({ changes: "", changedFields: [], metadata: "" })]) + const rows = yield* Effect.gen(function* () { + const audit = yield* AuditLogService + return yield* audit.list(ORG, { limit: 10, offset: 0 }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + expect(rows[0]!.changes).toBeNull() + expect(rows[0]!.changedFields).toBeNull() + expect(rows[0]!.metadata).toBeNull() + }), + ) + + it.effect("binds only the filters the caller set", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.list(ORG, { limit: 10, offset: 0 }) + yield* audit.list(ORG, { + actorType: "api_key", + changedField: "scopes", + sinceMs: Date.UTC(2026, 7, 29, 9, 12, 0, 412), + limit: 5, + offset: 5, + }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + const [plain, filtered] = warehouse.sql + expect(plain).toContain(`OrgId = '${ORG}'`) + expect(plain).not.toContain("ActorType =") + expect(plain).not.toContain("has(ChangedFields") + expect(plain).toMatch(/ORDER BY occurredAt DESC, id DESC/) + expect(filtered).toContain("ActorType = 'api_key'") + expect(filtered).toContain("has(ChangedFields, 'scopes')") + expect(filtered).toContain("OccurredAt >= '2026-08-29 09:12:00.412'") + expect(filtered).toMatch(/LIMIT 5\s+OFFSET 5/) + // Never routed to an org's BYO warehouse. + expect(plain).toContain("audit_log") + }), + ) +}) + +/** Three entries with distinct timestamps: user, then api_key, then agent. */ +const seedThree = Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + resourceId: DASHBOARD_ID, + metadata: { name: "First" }, + }) + yield* TestClock.adjust("1 second") + yield* audit.record({ + orgId: ORG, + actor: { type: "api_key" }, + source: "api", + action: "alert_rule.updated", + }) + yield* TestClock.adjust("1 second") + yield* audit.record({ + orgId: ORG, + actor: { type: "agent", label: "triage-bot" }, + source: "mcp", + action: "error_issue.state_change", + }) +}) + +// The in-memory layer backs every route and workflow test, so its filter and +// ordering semantics must match the warehouse query's. +describe("AuditLogService.layerMemory", () => { + it.effect("round-trips a recorded entry and lists newest first", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + + const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(rows.map((row) => row.action)).toEqual([ + "error_issue.state_change", + "alert_rule.updated", + "dashboard.created", + ]) + + const oldest = rows[2]! + expect(oldest.actorType).toBe("user") + expect(oldest.userId).toBe(USER) + expect(oldest.source).toBe("dashboard") + expect(oldest.resourceType).toBe("dashboard") + expect(oldest.resourceId).toBe(encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID)) + expect(oldest.metadata).toEqual({ name: "First" }) + + const newest = rows[0]! + expect(newest.actorType).toBe("agent") + expect(newest.actorLabel).toBe("triage-bot") + }).pipe(Effect.provide(AuditLogService.layerMemory)), + ) + + it.effect("filters by actor type and outcome, and pages newest-first", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "alert_rule.deleted", + outcome: "denied", + denialReason: "missing role: admin", + }) + + const apiKeyRows = yield* audit.list(ORG, { actorType: "api_key", limit: 10, offset: 0 }) + expect(apiKeyRows.map((row) => row.action)).toEqual(["alert_rule.updated"]) + expect(yield* audit.list(ORG, { actorType: "system", limit: 10, offset: 0 })).toEqual([]) + + const denied = yield* audit.list(ORG, { outcome: "denied", limit: 10, offset: 0 }) + expect(denied.map((row) => row.action)).toEqual(["alert_rule.deleted"]) + expect(denied[0]!.denialReason).toBe("missing role: admin") + + const secondPage = yield* audit.list(ORG, { limit: 2, offset: 2 }) + expect(secondPage.map((row) => row.action)).toEqual(["alert_rule.updated", "dashboard.created"]) + }).pipe(Effect.provide(AuditLogService.layerMemory)), + ) + + it.effect("stores update diffs and filters by changed field", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.updated", + changes: { fields: ["name"], before: { name: "a" }, after: { name: "b" } }, + }) + + const rows = yield* audit.list(ORG, { changedField: "name", limit: 10, offset: 0 }) + expect(rows.map((row) => row.action)).toEqual(["dashboard.updated"]) + expect(rows[0]!.changedFields).toEqual(["name"]) + expect(rows[0]!.changes).toEqual({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }) + expect(yield* audit.list(ORG, { changedField: "description", limit: 10, offset: 0 })).toEqual([]) + }).pipe(Effect.provide(AuditLogService.layerMemory)), + ) + + // The credential and the surface are the two facts a mutation handler cannot + // re-derive, and getting them wrong is what made API-key and MCP actions read + // back as dashboard sessions. + describe("recordHttpAudit attribution", () => { + const tenant = new CurrentTenant.TenantSchema({ + orgId: ORG, + userId: USER, + roles: [], + authMode: "self_hosted", + }) + + const recordAs = (info: AuditActorInfo | undefined) => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* recordHttpAudit("dashboard.created", { resourceId: DASHBOARD_ID }) + const rows = yield* audit.list(ORG, { limit: 1, offset: 0 }) + return rows[0]! + }).pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, info), + Effect.provide(AuditLogService.layerMemory), + ) + + it.effect("attributes an API-key request to the key, not the dashboard", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "api_key", apiKeyId: API_KEY, source: "api" }) + expect(row.actorType).toBe("api_key") + expect(row.source).toBe("api") + expect(row.apiKeyId).toBe(API_KEY) + }), + ) + + it.effect("records the MCP surface rather than assuming a dashboard session", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "api_key", source: "mcp" }) + expect(row.source).toBe("mcp") + expect(row.actorType).toBe("api_key") + }), + ) + + it.effect("records Maple's own internal-token actions as system", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "system", source: "system" }) + expect(row.actorType).toBe("system") + expect(row.source).toBe("system") + }), + ) + + it.effect("falls back to the tenant user when no middleware set the reference", () => + Effect.gen(function* () { + const row = yield* recordAs(undefined) + expect(row.actorType).toBe("user") + expect(row.source).toBe("dashboard") + expect(row.userId).toBe(USER) + expect(row.apiKeyId).toBeNull() + }), + ) + }) +}) diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts new file mode 100644 index 000000000..f563d96f3 --- /dev/null +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -0,0 +1,421 @@ +import { randomUUID } from "node:crypto" +import { HttpServerRequest } from "effect/unstable/http" +import { AuditLogPersistenceError, CurrentTenant } from "@maple/domain/http" +import type { AuditActorType, AuditChanges, AuditLogSource, AuditOutcome } from "@maple/domain/http" +import type { ActorId, ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import { AuditLogEntryId as AuditLogEntryIdSchema } from "@maple/domain/primitives" +import * as CH from "@maple/query-engine/ch" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" +import type { Queue } from "@cloudflare/workers-types" +import { WorkerEnvironment } from "@maple/infra/worker-runtime" +import { warehouseDateTime64 } from "@maple/query-engine/datetime" +import { systemTenant } from "@/services/alerts/system-tenant" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { type AuditAction, auditResourceFields, type AuditResourceIdOption } from "./audit-actions" +import { + AuditLogEvent, + type AuditLogEntry, + auditEventToEntry, + auditEventToRow, + decodeStoredAuditLogEntry, + encodeAuditLogEventSync, + storedRowToEntry, +} from "./audit-event" + +const decodeAuditLogEntryIdSync = Schema.decodeUnknownSync(AuditLogEntryIdSchema) + +class AuditQueueSendError extends Schema.TaggedError()( + "@maple/api/services/audit/AuditQueueSendError", + { + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + +/** Producer binding name; the paired `*_NAME` var drives consumer dispatch. */ +export const AUDIT_EVENTS_QUEUE_BINDING = "AUDIT_EVENTS_QUEUE" + +/** The warehouse datasource every audit entry lands in. */ +export const AUDIT_LOG_DATASOURCE = "audit_log" + +/** + * `queue.send` sits on the response path of every mutation, denial, and + * audited read, and is the only thing that does. A healthy send is tens of ms; + * 2s bounds a stalling broker, after which the entry is dropped with a warning + * rather than charged to the response as a second network round trip. + */ +export const AUDIT_QUEUE_SEND_TIMEOUT = "2 seconds" + +const toPersistenceError = (error: { readonly _tag: string; readonly message?: string }) => + new AuditLogPersistenceError({ message: error.message ?? error._tag, cause: error }) + +/** The credential-holder behind an audited action, as known at the call site. */ +export interface AuditActorRef { + readonly type: AuditActorType + readonly userId?: UserId + readonly apiKeyId?: ApiKeyId + readonly actorId?: ActorId + readonly label?: string +} + +export type AuditLogRecordInput = { + readonly orgId: OrgId + readonly actor: AuditActorRef + readonly source: AuditLogSource + /** Declared in `AuditResources`; the row's `resource_type` is derived from it. */ + readonly action: A + /** Defaults to `"allowed"`; denied attempts pass `"denied"` + `denialReason`. */ + readonly outcome?: AuditOutcome + readonly denialReason?: string + readonly affectedUserId?: UserId + readonly changes?: AuditChanges | undefined + readonly metadata?: Record + readonly requestId?: string + readonly originIp?: string + readonly originCountry?: string +} & AuditResourceIdOption + +export interface AuditLogListFilters { + readonly actorType?: AuditActorType + /** At most one of the three actor-identity filters is set per request. */ + readonly userId?: UserId + readonly apiKeyId?: ApiKeyId + readonly actorId?: ActorId + readonly affectedUserId?: UserId + readonly action?: string + readonly outcome?: AuditOutcome + readonly resourceType?: string + /** Matches the stored public form (e.g. `dash_…`). */ + readonly resourceId?: string + /** Field name that an update's diff must have touched. */ + readonly changedField?: string + readonly requestId?: string + readonly sinceMs?: number + readonly untilMs?: number + readonly limit: number + readonly offset: number +} + +export interface AuditLogServiceApi { + /** + * Append one entry: published to the audit events queue when the binding is + * present (the consumer performs the warehouse write, retried by the queue + * and parked in the DLQ after that), written straight to the warehouse only + * where there is no queue at all — local dev, crons, the consumer itself — + * none of which are serving a response. + * Never fails: an action that succeeded must not 500 because its audit + * write did not — terminal failures are logged and swallowed. + */ + readonly record: (input: AuditLogRecordInput) => Effect.Effect + readonly list: ( + orgId: OrgId, + filters: AuditLogListFilters, + ) => Effect.Effect, AuditLogPersistenceError> +} + +/** Build the queue event for one `record` call, stamping id and `occurredAt`. */ +const makeEvent = (input: AuditLogRecordInput, nowMs: number) => { + const resource = auditResourceFields(input.action, input.resourceId) + return new AuditLogEvent({ + orgId: input.orgId, + id: decodeAuditLogEntryIdSync(randomUUID()), + actorType: input.actor.type, + ...(input.actor.userId !== undefined ? { userId: input.actor.userId } : undefined), + ...(input.actor.apiKeyId !== undefined ? { apiKeyId: input.actor.apiKeyId } : undefined), + ...(input.actor.actorId !== undefined ? { actorId: input.actor.actorId } : undefined), + ...(input.actor.label !== undefined ? { actorLabel: input.actor.label } : undefined), + ...(input.affectedUserId !== undefined ? { affectedUserId: input.affectedUserId } : undefined), + source: input.source, + action: input.action, + outcome: input.outcome ?? "allowed", + ...(input.denialReason !== undefined ? { denialReason: input.denialReason } : undefined), + resourceType: resource.resourceType, + ...(resource.resourceId !== undefined ? { resourceId: resource.resourceId } : undefined), + ...(input.changes !== undefined ? { changes: input.changes } : undefined), + ...(input.metadata !== undefined ? { metadata: input.metadata } : undefined), + ...(input.requestId !== undefined ? { requestId: input.requestId } : undefined), + ...(input.originIp !== undefined ? { originIp: input.originIp } : undefined), + ...(input.originCountry !== undefined ? { originCountry: input.originCountry } : undefined), + occurredAtMs: nowMs, + }) +} + +/** Self-observability: refused attempts are the entries worth alerting on. */ +const logDenied = (event: AuditLogEvent) => + event.outcome === "denied" + ? Effect.logWarning("Audit: denied action").pipe( + Effect.annotateLogs({ + orgId: event.orgId, + action: event.action, + actorType: event.actorType, + denialReason: event.denialReason ?? "", + }), + ) + : Effect.void + +/** + * `record` never fails: swallow typed failures and defects — an action that + * succeeded must not 500 because its audit write did not — but let interrupts + * propagate so fiber teardown never triggers a stray write. + */ +const neverFail = (action: string) => (write: Effect.Effect) => + write.pipe( + Effect.catch((error) => Effect.logWarning("Audit log write failed", { action, cause: error })), + Effect.catchDefect((defect) => Effect.logWarning("Audit log write failed", { action, cause: defect })), + ) + +/** Which optional filters bind, and the parameter values behind them. */ +const listQueryInputs = (orgId: OrgId, filters: AuditLogListFilters) => { + const since = filters.sinceMs === undefined ? undefined : warehouseDateTime64(filters.sinceMs) + const until = filters.untilMs === undefined ? undefined : warehouseDateTime64(filters.untilMs) + const opts: CH.AuditLogEntriesOpts = { + actorType: filters.actorType !== undefined, + userId: filters.userId !== undefined, + apiKeyId: filters.apiKeyId !== undefined, + actorId: filters.actorId !== undefined, + affectedUserId: filters.affectedUserId !== undefined, + action: filters.action !== undefined, + outcome: filters.outcome !== undefined, + resourceType: filters.resourceType !== undefined, + resourceId: filters.resourceId !== undefined, + changedField: filters.changedField !== undefined, + requestId: filters.requestId !== undefined, + since: since !== undefined, + until: until !== undefined, + limit: filters.limit, + offset: filters.offset, + } + const values = { + orgId, + ...(filters.actorType !== undefined ? { actorType: filters.actorType } : undefined), + ...(filters.userId !== undefined ? { userId: filters.userId } : undefined), + ...(filters.apiKeyId !== undefined ? { apiKeyId: filters.apiKeyId } : undefined), + ...(filters.actorId !== undefined ? { actorId: filters.actorId } : undefined), + ...(filters.affectedUserId !== undefined ? { affectedUserId: filters.affectedUserId } : undefined), + ...(filters.action !== undefined ? { action: filters.action } : undefined), + ...(filters.outcome !== undefined ? { outcome: filters.outcome } : undefined), + ...(filters.resourceType !== undefined ? { resourceType: filters.resourceType } : undefined), + ...(filters.resourceId !== undefined ? { resourceId: filters.resourceId } : undefined), + ...(filters.changedField !== undefined ? { changedField: filters.changedField } : undefined), + ...(filters.requestId !== undefined ? { requestId: filters.requestId } : undefined), + ...(since !== undefined ? { since } : undefined), + ...(until !== undefined ? { until } : undefined), + } + return { opts, values } +} + +export class AuditLogService extends Context.Service()( + "@maple/api/services/AuditLogService", + { + make: Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + // Optional so tests and non-Worker runtimes fall back to direct writes + // without providing a WorkerEnvironment. + const workerEnv = yield* Effect.serviceOption(WorkerEnvironment) + const queue = Option.match(workerEnv, { + onNone: () => undefined, + onSome: (env) => { + const binding = env[AUDIT_EVENTS_QUEUE_BINDING] + // SAFETY: the binding slot is owned by this service; anything present is the queue. + return binding === undefined ? undefined : (binding as Queue) + }, + }) + + // One row through the managed ingest pipeline. `ingest` is pinned to + // Tinybird regardless of the org's read backend, which is the point: + // the audit log is Maple's record and never lands in a BYO warehouse. + const writeDirect = (event: AuditLogEvent) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + yield* warehouse.ingest(systemTenant(event.orgId), AUDIT_LOG_DATASOURCE, [ + auditEventToRow(event, now), + ]) + }) + + // Queue unavailability must not lose the entry: degrade to a direct + // write before giving up. Only typed send failures land here — an + // interrupt must propagate, not spawn a warehouse write mid-teardown. + /** + * One queue send, and that is the whole write path wherever a queue + * exists. It used to fall back to a direct warehouse write when the + * send failed — which put a second network round trip on the response + * path exactly when the platform was already degraded, turning a + * Queues brown-out into slow requests for every audited read. The + * queue is the durability story (retries, DLQ); if the send itself + * cannot be made, the entry is lost and says so in the logs rather + * than being bought at the caller's expense. + * + * `writeDirect` remains for runtimes with no queue binding at all — + * local dev, crons, and the consumer itself — none of which are + * serving a response. + */ + const publish = (event: AuditLogEvent) => + queue === undefined + ? writeDirect(event) + : Effect.tryPromise({ + try: () => queue.send(encodeAuditLogEventSync(event)), + catch: (cause) => + new AuditQueueSendError({ message: "Audit queue send failed", cause }), + }).pipe( + // A Queues brown-out that stalls rather than rejects must not + // hang the response: 2s is far above a healthy send's latency + // yet bounds the worst case. + Effect.timeout(AUDIT_QUEUE_SEND_TIMEOUT), + Effect.catchTag("TimeoutError", (error) => + Effect.fail( + new AuditQueueSendError({ message: "Audit queue send timed out", cause: error }), + ), + ), + ) + + const record: AuditLogServiceApi["record"] = Effect.fn("AuditLogService.record")(function* ( + input, + ) { + const now = yield* Clock.currentTimeMillis + const event = makeEvent(input, now) + yield* logDenied(event) + yield* publish(event).pipe(neverFail(input.action)) + }) + + const list: AuditLogServiceApi["list"] = Effect.fn("AuditLogService.list")(function* ( + orgId, + filters, + ) { + const { opts, values } = listQueryInputs(orgId, filters) + const rows = yield* warehouse + .compiledQuery( + systemTenant(orgId), + CH.compile(CH.auditLogEntriesQuery(opts), values), + { profile: "list", context: "auditLog.list" }, + ) + .pipe(Effect.mapError(toPersistenceError)) + const entries = yield* Effect.forEach(rows, (row) => + decodeStoredAuditLogEntry(row).pipe( + Effect.map((decoded) => storedRowToEntry(orgId, decoded)), + Effect.mapError((error) => + new AuditLogPersistenceError({ + message: "Stored audit entry failed to decode", + cause: error, + }), + ), + ), + ) + // A redelivered event the ReplacingMergeTree has not merged yet is + // the same entry twice; one copy is enough. + const seen = new Set() + return entries.filter((entry) => !seen.has(entry.id) && seen.add(entry.id) !== undefined) + }) + + return { record, list } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) + + /** + * Entries kept in process memory, with the same filter and ordering + * semantics as the warehouse query. For tests and anything else that must + * not reach a warehouse. + */ + static readonly layerMemory = Layer.sync(this, makeMemoryAuditLog) +} + +/** The in-memory implementation behind `AuditLogService.layerMemory`; usable directly in a `Context`. */ +export function makeMemoryAuditLog(): AuditLogServiceApi { + const entries: Array = [] + const record: AuditLogServiceApi["record"] = (input) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const event = makeEvent(input, now) + yield* logDenied(event) + entries.push(auditEventToEntry(event, now)) + }) + const list: AuditLogServiceApi["list"] = (orgId, filters) => + Effect.succeed( + entries + .filter( + (entry) => + entry.orgId === orgId && + (filters.actorType === undefined || entry.actorType === filters.actorType) && + (filters.userId === undefined || entry.userId === filters.userId) && + (filters.apiKeyId === undefined || entry.apiKeyId === filters.apiKeyId) && + (filters.actorId === undefined || entry.actorId === filters.actorId) && + (filters.affectedUserId === undefined || + entry.affectedUserId === filters.affectedUserId) && + (filters.action === undefined || entry.action === filters.action) && + (filters.outcome === undefined || entry.outcome === filters.outcome) && + (filters.resourceType === undefined || entry.resourceType === filters.resourceType) && + (filters.resourceId === undefined || entry.resourceId === filters.resourceId) && + (filters.changedField === undefined || + (entry.changedFields?.includes(filters.changedField) ?? false)) && + (filters.requestId === undefined || entry.requestId === filters.requestId) && + (filters.sinceMs === undefined || entry.occurredAt.getTime() >= filters.sinceMs) && + (filters.untilMs === undefined || entry.occurredAt.getTime() <= filters.untilMs), + ) + .sort( + (a, b) => + b.occurredAt.getTime() - a.occurredAt.getTime() || (b.id < a.id ? -1 : b.id > a.id ? 1 : 0), + ) + .slice(filters.offset, filters.offset + filters.limit), + ) + return { record, list } +} + +/** Request forensics for an audit entry, read off the Cloudflare request headers. */ +export const httpRequestForensics = (request: HttpServerRequest.HttpServerRequest) => ({ + ...(request.headers["cf-ray"] !== undefined ? { requestId: request.headers["cf-ray"] } : undefined), + ...(request.headers["cf-connecting-ip"] !== undefined + ? { originIp: request.headers["cf-connecting-ip"] } + : undefined), + ...(request.headers["cf-ipcountry"] !== undefined + ? { originCountry: request.headers["cf-ipcountry"] } + : undefined), +}) + +/** Forensics for the current request, or nothing outside an HTTP request. */ +export const currentRequestForensics = Effect.gen(function* () { + const request = yield* Effect.serviceOption(HttpServerRequest.HttpServerRequest) + return Option.match(request, { + onNone: () => ({}), + onSome: httpRequestForensics, + }) +}) + +/** + * Record an audit entry for the current authenticated HTTP request, deriving + * the actor and surface from the tenant plus the auth middleware's + * `CurrentAuditActor`, and request forensics (request id, origin) from the + * Cloudflare headers. The credential kind and the surface both come from the + * reference — an API-key or MCP request must not read back as a dashboard + * session. + */ +export const recordHttpAudit = ( + action: A, + opts?: { + readonly changes?: AuditChanges | undefined + readonly affectedUserId?: UserId + readonly metadata?: Record + } & AuditResourceIdOption, +) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const tenant = yield* CurrentTenant.Context + const info = yield* CurrentAuditActor + const context = yield* currentRequestForensics + // No reference means the request bypassed every auth middleware (internal + // tokens, tests). Attribute to the tenant's user rather than inventing a + // credential, but do not claim a surface the request may not have used. + yield* audit.record({ + orgId: tenant.orgId, + actor: { + type: info?.type ?? "user", + userId: tenant.userId, + ...(info?.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + }, + source: info?.source ?? "dashboard", + action, + ...context, + ...opts, + }) + }) diff --git a/apps/api/src/services/audit/audit-access.test.ts b/apps/api/src/services/audit/audit-access.test.ts new file mode 100644 index 000000000..9aec60ee1 --- /dev/null +++ b/apps/api/src/services/audit/audit-access.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "@effect/vitest" +import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { CurrentTenant, MapleInternalApi } from "@maple/domain/http" +import { MapleApiV2 } from "@maple/domain/http/v2" +import type { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" +import { OrgId, UserId } from "@maple/domain/primitives" +import { Effect, Result, Schema } from "effect" +import { TestClock } from "effect/testing" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { makeMemoryAuditLog } from "./AuditLogService" +import { auditAttribution, recordRawSqlAudit, withAuditedRead } from "./audit-access" +import { AuditLogService } from "./AuditLogService" + +const ORG = Schema.decodeUnknownSync(OrgId)("org_audit_access_test") +const USER = Schema.decodeUnknownSync(UserId)("user_audit_access_test") + +/** The `{ group, endpoint }` a security middleware receives for one endpoint. */ +const endpointOf = (api: { readonly groups: Record }, group: string, name: string) => { + const found = api.groups[group] + if (found === undefined) throw new Error(`no group ${group}`) + const endpoint = found.endpoints[name] as HttpApiEndpoint.Top | undefined + if (endpoint === undefined) throw new Error(`no endpoint ${group}.${name}`) + return { group: found, endpoint } +} + +const request = (method: string, url: string, body?: string) => + HttpServerRequest.fromWeb( + new Request(`https://api.test${url}`, { + method, + headers: { "cf-ray": "ray-1", "cf-connecting-ip": "203.0.113.7" }, + ...(body !== undefined ? { body } : undefined), + }), + ) + +class HandlerFailure extends Schema.TaggedError()("HandlerFailure", { + message: Schema.String, +}) {} + +const subject = { + orgId: ORG, + actor: { type: "user" as const, userId: USER }, + source: "dashboard" as const, +} + +describe("withAuditedRead", () => { + it.effect("records a telemetry read for an endpoint whose GROUP carries the annotation", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const req = request("POST", "/internal/query-engine/execute-batch?x=1", '{"requests":[]}') + const options = endpointOf(MapleInternalApi, "queryEngine", "executeBatch") + // The handler reads the body first, exactly as a real one would. + const handler = req.text.pipe(Effect.map(() => HttpServerResponse.empty({ status: 200 }))) + yield* withAuditedRead(audit, req, options, subject)(handler) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries).toHaveLength(1) + const entry = entries[0]! + expect(entry.action).toBe("telemetry.read") + expect(entry.userId).toBe(USER) + expect(entry.requestId).toBe("ray-1") + expect(entry.metadata).toMatchObject({ + endpoint: "queryEngine.executeBatch", + method: "POST", + status: 200, + body: '{"requests":[]}', + }) + }), + ) + + it.effect("records session replay reads on the v2 group and nothing for unannotated endpoints", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const ok = Effect.succeed(HttpServerResponse.empty({ status: 200 })) + yield* withAuditedRead(audit, request("GET", "/v2/session_replays/s1"), endpointOf(MapleApiV2, "sessionReplays", "retrieve"), subject)(ok) + yield* withAuditedRead(audit, request("GET", "/v2/api_keys"), endpointOf(MapleApiV2, "apiKeys", "list"), subject)(ok) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries.map((entry) => entry.action)).toEqual(["session_replay.read"]) + }), + ) + + it.effect("still records an attempted read when the handler fails", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const failing = Effect.fail(new HandlerFailure({ message: "boom" })) + const outcome = yield* withAuditedRead( + audit, + request("GET", "/v2/traces/t1"), + endpointOf(MapleApiV2, "traces", "retrieve"), + subject, + )(failing).pipe(Effect.result) + expect(Result.isFailure(outcome)).toBe(true) + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries).toHaveLength(1) + expect(entries[0]!.metadata).toMatchObject({ status: 0, endpoint: "traces.retrieve" }) + }), + ) +}) + +describe("auditAttribution", () => { + it("attributes an agent tenant to the agent acting for the user", () => { + const actorId = Schema.decodeUnknownSync(Schema.String)("actor_1") + const attribution = auditAttribution( + { orgId: ORG, userId: USER, actorId: actorId as never, mcpClientName: "claude-code" }, + { type: "api_key", source: "mcp" }, + ) + expect(attribution).toEqual({ + actor: { type: "agent", actorId, userId: USER, label: "claude-code" }, + source: "mcp", + }) + }) + + it("freezes the API key's name onto the entry", () => { + const apiKeyId = Schema.decodeUnknownSync(Schema.String)("key_1") + expect( + auditAttribution( + { orgId: ORG, userId: USER }, + { type: "api_key", apiKeyId: apiKeyId as never, label: "Deploy bot", source: "api" }, + ), + ).toEqual({ + actor: { type: "api_key", userId: USER, apiKeyId, label: "Deploy bot" }, + source: "api", + }) + }) + + it("leaves a dashboard session unlabelled — its name is resolved when the log is read", () => { + expect(auditAttribution({ orgId: ORG, userId: USER }, { type: "user", source: "dashboard" })).toEqual({ + actor: { type: "user", userId: USER }, + source: "dashboard", + }) + }) + + it("keeps a system token as system regardless of the tenant", () => { + expect(auditAttribution({ orgId: ORG, userId: USER }, { type: "system", source: "system" })).toEqual({ + actor: { type: "system" }, + source: "system", + }) + }) +}) + +describe("recordRawSqlAudit", () => { + it.effect("records a refused statement as denied and an executed one with its row count", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + const base = { + tenant: { orgId: ORG, userId: USER }, + sql: "SELECT 1", + context: "mcp.run_sql", + startTime: "2026-08-29 09:00:00", + endTime: "2026-08-29 10:00:00", + } + yield* recordRawSqlAudit({ ...base, result: { _tag: "rejected", reason: "missing $__orgFilter" } }) + yield* TestClock.adjust("1 second") + yield* recordRawSqlAudit({ ...base, result: { _tag: "rows", rowCount: 3 } }) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries.map((entry) => [entry.action, entry.outcome])).toEqual([ + ["telemetry.sql_executed", "allowed"], + ["telemetry.sql_executed", "denied"], + ]) + expect(entries[1]!.denialReason).toBe("missing $__orgFilter") + expect(entries[0]!.metadata).toMatchObject({ sql: "SELECT 1", row_count: 3, context: "mcp.run_sql" }) + expect(entries[0]!.source).toBe("mcp") + }).pipe( + Effect.provideService(CurrentAuditActor, { type: "api_key", source: "mcp" }), + Effect.provide(AuditLogService.layerMemory), + ), + ) +}) diff --git a/apps/api/src/services/audit/audit-access.ts b/apps/api/src/services/audit/audit-access.ts new file mode 100644 index 000000000..0eac00e86 --- /dev/null +++ b/apps/api/src/services/audit/audit-access.ts @@ -0,0 +1,228 @@ +import { Context, Effect } from "effect" +import type { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import type { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" +import { AuditedRead, type AuditLogSource } from "@maple/domain/http" +import type { ActorId, OrgId, UserId } from "@maple/domain/primitives" +import type { McpToolSurface } from "@/mcp/dispatcher" +import type { AuditActorInfo } from "@/services/auth/audit-actor" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import type { TenantContext } from "@/services/auth/tenant-context" +import { + type AuditActorRef, + AuditLogService, + type AuditLogServiceApi, + currentRequestForensics, + httpRequestForensics, +} from "./AuditLogService" + +/** + * Access auditing: who read what. HIPAA's audit-control standard covers every + * access to protected data, not only changes, so three read surfaces record + * entries alongside the mutation trail — + * + * - HTTP endpoints annotated `AuditedRead` (telemetry and session replays), + * wrapped by the auth layers via {@link withAuditedRead}; + * - every MCP tool invocation, from the executor via {@link recordMcpToolAudit}; + * - every raw SQL statement, via {@link recordRawSqlAudit}. + */ + +/** Bound on stored request/parameter snapshots — enough to see what was asked for. */ +const MAX_SNAPSHOT_CHARS = 2_000 + +/** A JSON rendering of `value` capped at {@link MAX_SNAPSHOT_CHARS}. */ +export const snapshot = (value: unknown): string => { + const text = typeof value === "string" ? value : (JSON.stringify(value) ?? "null") + return text.length > MAX_SNAPSHOT_CHARS ? `${text.slice(0, MAX_SNAPSHOT_CHARS)}…` : text +} + +export interface AuditAttribution { + readonly actor: AuditActorRef + readonly source: AuditLogSource +} + +/** + * Attribute an action performed under `tenant`. The credential and surface + * come from the auth layer's `CurrentAuditActor` when one set it; an agent + * tenant (pinned `actorId`) is recorded as the agent acting on the user's + * behalf; nothing set means a dashboard session, never a guessed credential. + */ +/** The tenant facts attribution needs; both `TenantContext` and `TenantSchema` satisfy it. */ +export interface AuditTenant { + readonly orgId: OrgId + readonly userId: UserId + readonly actorId?: ActorId | undefined + readonly mcpClientName?: string | undefined +} + +export const auditAttribution = (tenant: AuditTenant, info: AuditActorInfo | undefined): AuditAttribution => { + if (info?.type === "system") return { actor: { type: "system" }, source: "system" } + if (tenant.actorId !== undefined) { + return { + actor: { + type: "agent", + actorId: tenant.actorId, + userId: tenant.userId, + ...(tenant.mcpClientName !== undefined ? { label: tenant.mcpClientName } : undefined), + }, + source: info?.source ?? "mcp", + } + } + return { + actor: { + type: info?.type ?? "user", + userId: tenant.userId, + ...(info?.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + ...(info?.label !== undefined ? { label: info.label } : undefined), + }, + source: info?.source ?? "dashboard", + } +} + +export interface AuditedReadSubject extends AuditAttribution { + readonly orgId: OrgId +} + +/** + * Wrap an authenticated endpoint response so that, when the endpoint is + * annotated `AuditedRead`, the call is recorded once it completes — with the + * HTTP status, the route, the request path, and (for POST searches) a bounded + * snapshot of the body that says what was queried. A typed failure still + * records an attempt; an interrupt records nothing. + */ +export const withAuditedRead = + ( + audit: AuditLogServiceApi, + request: HttpServerRequest.HttpServerRequest, + options: { readonly endpoint: HttpApiEndpoint.Top; readonly group: HttpApiGroup.Top }, + subject: AuditedReadSubject, + ) => + ( + httpEffect: Effect.Effect, + ): Effect.Effect => { + // A group-level `.annotate` lands on the group only (endpoint propagation + // is `annotateEndpoints`), so both are consulted; the endpoint wins. + const action = + Context.get(options.endpoint.annotations, AuditedRead) ?? + Context.get(options.group.annotations, AuditedRead) + if (action === undefined) return httpEffect + const record = (status: number) => + Effect.gen(function* () { + // The handler already consumed (and cached) the body, so this is a + // read of the same text, never a second parse of the stream. + const body = + request.method === "GET" || request.method === "HEAD" + ? undefined + : yield* request.text.pipe(Effect.option) + yield* audit.record({ + orgId: subject.orgId, + actor: subject.actor, + source: subject.source, + action, + metadata: { + endpoint: `${options.group.identifier}.${options.endpoint.identifier}`, + method: request.method, + path: request.url, + status, + ...(body !== undefined && body._tag === "Some" && body.value !== "" + ? { body: snapshot(body.value) } + : undefined), + }, + ...httpRequestForensics(request), + }) + }) + return httpEffect.pipe( + Effect.tap((response) => record(response.status)), + // A rejected read (bad parameters, not found) is still an attempt; + // 0 says the status was never produced. + Effect.tapError(() => record(0)), + ) + } + +export interface McpToolAuditInput { + readonly tenant: TenantContext + readonly name: string + readonly input: unknown + readonly surface: McpToolSurface + readonly isError: boolean +} + +/** + * One `mcp_tool.called` entry per tool invocation, whichever surface drove it. + * Workflow passes and internal RPC run under Maple's own tenant, so they are + * `system`; the public transport and the chat attribute through the tenant. + */ +export const recordMcpToolAudit = (input: McpToolAuditInput) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const info = yield* CurrentAuditActor + const forensics = yield* currentRequestForensics + const attribution = + input.surface === "workflow" || input.surface === "rpc" + ? { actor: { type: "system" as const, label: input.surface }, source: "system" as const } + : auditAttribution(input.tenant, info) + yield* audit.record({ + orgId: input.tenant.orgId, + ...attribution, + action: "mcp_tool.called", + metadata: { + tool: input.name, + surface: input.surface, + is_error: input.isError, + params: snapshot(input.input), + }, + ...forensics, + }) + }) + +export type RawSqlAuditResult = + | { readonly _tag: "rows"; readonly rowCount: number } + /** The safety pass refused the statement before it ran. */ + | { readonly _tag: "rejected"; readonly reason: string } + /** The warehouse refused or failed the statement. */ + | { readonly _tag: "failed"; readonly error: string } + +export interface RawSqlAuditInput { + readonly tenant: AuditTenant + readonly sql: string + /** The executor context label: `mcp.run_sql`, `rawSql`, … */ + readonly context: string + readonly startTime: string + readonly endTime: string + readonly result: RawSqlAuditResult +} + +/** + * One `telemetry.sql_executed` entry per raw SQL statement — the statement + * itself, the window it ran over, and how it ended. A statement the safety + * pass refused is a `denied` entry: an attempt to read outside the guardrails + * is exactly what an auditor asks about. + */ +export const recordRawSqlAudit = (input: RawSqlAuditInput) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const info = yield* CurrentAuditActor + const forensics = yield* currentRequestForensics + const { actor, source } = auditAttribution(input.tenant, info) + yield* audit.record({ + orgId: input.tenant.orgId, + actor, + source, + action: "telemetry.sql_executed", + ...(input.result._tag === "rejected" + ? { outcome: "denied", denialReason: input.result.reason } + : undefined), + metadata: { + sql: snapshot(input.sql), + context: input.context, + start_time: input.startTime, + end_time: input.endTime, + ...(input.result._tag === "rows" ? { row_count: input.result.rowCount } : undefined), + ...(input.result._tag === "failed" ? { error: input.result.error } : undefined), + }, + ...forensics, + }) + }) + +/** A one-line description of a typed failure for the audit metadata. */ +export const describeFailure = (error: { readonly _tag: string; readonly message?: string }): string => + error.message ?? error._tag diff --git a/apps/api/src/services/audit/audit-actions.test.ts b/apps/api/src/services/audit/audit-actions.test.ts new file mode 100644 index 000000000..b72044316 --- /dev/null +++ b/apps/api/src/services/audit/audit-actions.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest" +import { decodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" +import { ErrorIssueEventType } from "@maple/domain/http" +import { AuditResources, auditResourceFields } from "./audit-actions" + +describe("AuditResources", () => { + it("names every resource in the `.` snake_case shape the rows store", () => { + for (const [resource, { verbs }] of Object.entries(AuditResources)) { + expect(resource).toMatch(/^[a-z][a-z0-9_]*$/) + for (const verb of verbs) expect(verb).toMatch(/^[a-z][a-z0-9_]*$/) + } + }) + + // The issue workflow audits `error_issue.${type}` for every event type it + // attributes, so a new event type must not silently produce an undeclared action. + it("declares an `error_issue` verb for every issue event type", () => { + expect([...AuditResources.error_issue.verbs]).toEqual([...ErrorIssueEventType.literals]) + }) +}) + +describe("auditResourceFields", () => { + it("derives the resource type from the action", () => { + expect(auditResourceFields("alert_rule.created").resourceType).toBe("alert_rule") + expect(auditResourceFields("dashboard_share.rotated").resourceType).toBe("dashboard_share") + expect(auditResourceFields("dashboard.version_restored").resourceType).toBe("dashboard") + }) + + it("encodes the internal ID with the resource's own public prefix", () => { + const internal = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" + const { resourceId } = auditResourceFields("alert_rule.created", internal) + expect(resourceId).toMatch(/^alrt_/) + expect(decodePublicId(PublicIdPrefixes.alertRule, resourceId!)).toBe(internal) + }) + + it("omits the resource ID for org-singleton resources", () => { + expect(auditResourceFields("ingest_key.rolled")).toEqual({ resourceType: "ingest_key" }) + expect(auditResourceFields("api.request")).toEqual({ resourceType: "api" }) + }) +}) diff --git a/apps/api/src/services/audit/audit-actions.ts b/apps/api/src/services/audit/audit-actions.ts new file mode 100644 index 000000000..2a6761421 --- /dev/null +++ b/apps/api/src/services/audit/audit-actions.ts @@ -0,0 +1,136 @@ +import { encodePublicId, type PublicIdPrefix, PublicIdPrefixes } from "@maple/domain/http/v2" +import { ErrorIssueEventType } from "@maple/domain/http" + +/** + * Every audited action in Maple, grouped by the resource it acts on. + * + * The key is both the `resource_type` stored on the row and the `` + * half of the `.` action string, so the two can never disagree. + * `prefix` is the public-ID prefix the resource's internal ID is encoded with; + * resources that are org-singletons (`ingest_key`, `anomaly_settings`) or carry + * no resource at all (`api`) omit it, and passing a `resourceId` for one of + * those is a type error. + * + * Adding an entry here is what makes `record({ action: "." })` + * compile — a typo, or an action recorded before it is declared, fails the build. + */ +export const AuditResources = { + agent: { prefix: PublicIdPrefixes.actor, verbs: ["registered"] }, + alert_destination: { + prefix: PublicIdPrefixes.alertDestination, + verbs: ["created", "updated", "deleted"], + }, + alert_rule: { prefix: PublicIdPrefixes.alertRule, verbs: ["created", "updated", "deleted"] }, + anomaly_incident: { prefix: PublicIdPrefixes.anomalyIncident, verbs: ["resolved"] }, + /** Org-singleton settings — no resource id. */ + anomaly_settings: { verbs: ["updated"] }, + /** Refused requests, recorded by the auth layers; the route is in `metadata`. */ + api: { verbs: ["request"] }, + api_key: { prefix: PublicIdPrefixes.apiKey, verbs: ["created", "rolled", "revoked"] }, + attribute_mapping: { + prefix: PublicIdPrefixes.attributeMapping, + verbs: ["created", "updated", "deleted"], + }, + dashboard: { + prefix: PublicIdPrefixes.dashboard, + verbs: ["created", "updated", "deleted", "version_restored"], + }, + dashboard_share: { prefix: PublicIdPrefixes.dashboardShare, verbs: ["created", "rotated", "deleted"] }, + /** Verbs mirror the issue event types — `recordEvent` audits every one it attributes. */ + error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, + /** Org-singleton public/private pair; which one rolled is in `metadata`. */ + ingest_key: { verbs: ["rolled"] }, + /** + * Every MCP tool invocation, whichever surface drove it (MCP transport, the + * in-app chat, workflows, internal RPC). The tool and its parameters are in + * `metadata`; a tool that also mutates a resource records that action too. + */ + mcp_tool: { verbs: ["called"] }, + investigation: { prefix: PublicIdPrefixes.investigation, verbs: ["created", "restarted", "status_changed"] }, + /** + * Org-singleton connections. `*_started` is the admin action Maple sees; the + * OAuth round trip completes at the provider's callback. + */ + planetscale_integration: { + verbs: ["connect_started", "organization_selected", "metrics_token_set", "disconnected"], + }, + slack_integration: { verbs: ["install_started", "uninstalled"] }, + /** + * Org membership, learned from Clerk's webhook — the web app changes members + * in Clerk directly, so nothing reaches Maple's own API. The member is the + * entry's `affected_user`; no prefix, since Clerk IDs are already public. + */ + member: { verbs: ["added", "role_changed", "removed"] }, + /** + * The org itself. No prefix: every row already carries `org_id`, and a + * deleted org has no public ID left to resolve. + */ + organization: { verbs: ["deleted"] }, + scrape_target: { prefix: PublicIdPrefixes.scrapeTarget, verbs: ["created", "updated", "deleted"] }, + /** + * Reads of recorded browser sessions — the surface most likely to carry + * end-user data. Recorded by the auth layers from the `AuditedRead` annotation. + */ + session_replay: { verbs: ["read"] }, + /** + * Reads of traces, logs, metrics and error events (`read`, from the + * `AuditedRead` annotation on the endpoint) and every raw SQL statement run + * against the warehouse (`sql_executed`, with the statement in `metadata`). + */ + telemetry: { verbs: ["read", "sql_executed"] }, + /** Org-singleton BYO-ClickHouse connection; holds warehouse credentials. */ + warehouse_settings: { verbs: ["updated", "deleted", "schema_applied"] }, + /** Short-lived device credentials for the mobile widget; keyed by installation. */ + widget_credential: { verbs: ["minted", "revoked"] }, +} as const satisfies Record + +interface AuditResourceDefinition { + readonly prefix?: PublicIdPrefix + readonly verbs: ReadonlyArray +} + +export type AuditResourceType = keyof typeof AuditResources + +/** `.` for every declared pair — the closed set of audit actions. */ +export type AuditAction = { + [K in AuditResourceType]: `${K}.${(typeof AuditResources)[K]["verbs"][number]}` +}[AuditResourceType] + +type ResourceOf = A extends `${infer R}.${string}` + ? R extends AuditResourceType + ? R + : never + : never + +/** + * The `resourceId` option for an action: the resource's *internal* ID, encoded + * to its public `_…` form on the way to the row. Resources that declare + * no prefix (org-singletons) accept no `resourceId` at all. + */ +export type AuditResourceIdOption = (typeof AuditResources)[ResourceOf] extends { + readonly prefix: PublicIdPrefix +} + ? { readonly resourceId?: string } + : { readonly resourceId?: never } + +/** + * Derive the row's `resource_type` from the action and encode the internal + * resource ID into its public form, so no call site restates either. + */ +export const auditResourceFields = ( + action: AuditAction, + resourceId?: string, +): { readonly resourceType: AuditResourceType; readonly resourceId?: string } => { + // SAFETY: every `AuditAction` is built as `${resource}.${verb}` from the keys + // of `AuditResources`, so the segment before the dot is always one of them. + const resourceType = action.slice(0, action.indexOf(".")) as AuditResourceType + const resource = AuditResources[resourceType] + // Narrow rather than widen: org-singleton resources declare no `prefix` at all. + const prefix = "prefix" in resource ? resource.prefix : undefined + return { + resourceType, + ...(resourceId !== undefined && prefix !== undefined + ? { resourceId: encodePublicId(prefix, resourceId) } + : undefined), + } +} diff --git a/apps/api/src/services/audit/audit-event.ts b/apps/api/src/services/audit/audit-event.ts new file mode 100644 index 000000000..f5f0f2a4b --- /dev/null +++ b/apps/api/src/services/audit/audit-event.ts @@ -0,0 +1,201 @@ +import { + AuditActorType, + AuditChanges, + AuditLogSource, + AuditOutcome, +} from "@maple/domain/http" +import { ActorId, ApiKeyId, AuditLogEntryId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditLogRow } from "@maple/domain/tinybird" +import { Schema, SchemaTransformation } from "effect" +import { warehouseDateTime64 } from "@maple/query-engine/datetime" +import { msToDate } from "@/platform/time" + +/** + * The serialized audit event as it travels the audit queue. `occurredAtMs` is + * stamped by the producer; `recordedAt` exists only on the stored row, stamped + * by whichever writer performs the insert. + */ +export class AuditLogEvent extends Schema.Class("AuditLogEvent")({ + orgId: OrgId, + id: AuditLogEntryId, + actorType: AuditActorType, + userId: Schema.optionalKey(UserId), + apiKeyId: Schema.optionalKey(ApiKeyId), + actorId: Schema.optionalKey(ActorId), + actorLabel: Schema.optionalKey(Schema.String), + affectedUserId: Schema.optionalKey(UserId), + source: AuditLogSource, + action: Schema.String, + outcome: AuditOutcome, + denialReason: Schema.optionalKey(Schema.String), + resourceType: Schema.optionalKey(Schema.String), + resourceId: Schema.optionalKey(Schema.String), + changes: Schema.optionalKey(AuditChanges), + metadata: Schema.optionalKey(Schema.Record(Schema.String, Schema.Unknown)), + requestId: Schema.optionalKey(Schema.String), + originIp: Schema.optionalKey(Schema.String), + originCountry: Schema.optionalKey(Schema.String), + occurredAtMs: Schema.Finite, +}) {} + +export const decodeAuditLogEvent = Schema.decodeUnknownEffect(AuditLogEvent) +export const encodeAuditLogEventSync = Schema.encodeSync(AuditLogEvent) + +/** + * One stored audit entry, as the service hands it to readers. Absent values are + * `null` here and `''` in the warehouse row; the two lowering functions below + * are the only places that mapping lives. + */ +export interface AuditLogEntry { + readonly orgId: OrgId + readonly id: AuditLogEntryId + readonly actorType: AuditActorType + readonly userId: UserId | null + readonly apiKeyId: ApiKeyId | null + readonly actorId: ActorId | null + readonly actorLabel: string | null + readonly affectedUserId: UserId | null + readonly source: AuditLogSource + readonly action: string + readonly outcome: AuditOutcome + readonly denialReason: string | null + readonly resourceType: string | null + readonly resourceId: string | null + readonly changedFields: ReadonlyArray | null + readonly changes: AuditChanges | null + readonly metadata: Record | null + readonly requestId: string | null + readonly originIp: string | null + readonly originCountry: string | null + readonly occurredAt: Date + readonly recordedAt: Date +} + +/** Lower a queue event to its `audit_log` warehouse row; `recordedAtMs` is the write time. */ +export const auditEventToRow = (event: AuditLogEvent, recordedAtMs: number): AuditLogRow => ({ + OrgId: event.orgId, + Id: event.id, + OccurredAt: warehouseDateTime64(event.occurredAtMs), + RecordedAt: warehouseDateTime64(recordedAtMs), + ActorType: event.actorType, + UserId: event.userId ?? "", + ApiKeyId: event.apiKeyId ?? "", + ActorId: event.actorId ?? "", + ActorLabel: event.actorLabel ?? "", + AffectedUserId: event.affectedUserId ?? "", + Source: event.source, + Action: event.action, + Outcome: event.outcome, + DenialReason: event.denialReason ?? "", + ResourceType: event.resourceType ?? "", + ResourceId: event.resourceId ?? "", + ChangedFields: event.changes === undefined ? [] : [...event.changes.fields], + Changes: event.changes === undefined ? "" : JSON.stringify(event.changes), + Metadata: event.metadata === undefined ? "" : JSON.stringify(event.metadata), + RequestId: event.requestId ?? "", + OriginIp: event.originIp ?? "", + OriginCountry: event.originCountry ?? "", +}) + +/** The entry a reader would get back for `event` — what the in-memory layer stores. */ +export const auditEventToEntry = (event: AuditLogEvent, recordedAtMs: number): AuditLogEntry => ({ + orgId: event.orgId, + id: event.id, + actorType: event.actorType, + userId: event.userId ?? null, + apiKeyId: event.apiKeyId ?? null, + actorId: event.actorId ?? null, + actorLabel: event.actorLabel ?? null, + affectedUserId: event.affectedUserId ?? null, + source: event.source, + action: event.action, + outcome: event.outcome, + denialReason: event.denialReason ?? null, + resourceType: event.resourceType ?? null, + resourceId: event.resourceId ?? null, + changedFields: event.changes === undefined ? null : [...event.changes.fields], + changes: event.changes ?? null, + metadata: event.metadata ?? null, + requestId: event.requestId ?? null, + originIp: event.originIp ?? null, + originCountry: event.originCountry ?? null, + occurredAt: msToDate(event.occurredAtMs), + recordedAt: msToDate(recordedAtMs), +}) + +const JsonRecord = Schema.Record(Schema.String, Schema.Unknown) + +/** `''` in the warehouse row is "absent"; everything else decodes through `schema`. */ +const emptyAsNull = >(schema: S) => + Schema.String.pipe( + Schema.decodeTo( + Schema.NullOr(Schema.String), + SchemaTransformation.transform({ + decode: (value: string) => (value === "" ? null : value), + encode: (value: string | null) => value ?? "", + }), + ), + Schema.decodeTo(Schema.NullOr(schema)), + ) + +const nullableText = emptyAsNull(Schema.String) + +/** JSON document columns: `''` when absent, otherwise a JSON string of `schema`. */ +const jsonDocument = (schema: S) => emptyAsNull(Schema.fromJsonString(schema)) + +/** + * `YYYY-MM-DD HH:mm:ss.SSS` (UTC, as the warehouse emits DateTime64) ⇄ `Date`; + * an ISO rendering with `T`/`Z` is accepted as-is should a backend emit one. + */ +const warehouseDateTime64Column = Schema.String.pipe( + Schema.decodeTo( + Schema.Date, + SchemaTransformation.transform({ + decode: (value: string) => new Date(/[TZ]/.test(value) ? value : `${value.replace(" ", "T")}Z`), + // The brand is the minting side's guarantee; a codec encodes to the + // wire type, which is a plain string. + encode: (value: Date): string => warehouseDateTime64(value.getTime()), + }), + ), +) + +/** + * A listed row exactly as the warehouse returns it, with the `''`-means-absent + * convention decoded back to `null` and the JSON document columns parsed. + */ +export const StoredAuditLogEntry = Schema.Struct({ + id: AuditLogEntryId, + occurredAt: warehouseDateTime64Column, + recordedAt: warehouseDateTime64Column, + actorType: AuditActorType, + userId: emptyAsNull(UserId), + apiKeyId: emptyAsNull(ApiKeyId), + actorId: emptyAsNull(ActorId), + actorLabel: nullableText, + affectedUserId: emptyAsNull(UserId), + source: AuditLogSource, + action: Schema.String, + outcome: AuditOutcome, + denialReason: nullableText, + resourceType: nullableText, + resourceId: nullableText, + changedFields: Schema.Array(Schema.String), + changes: jsonDocument(AuditChanges), + metadata: jsonDocument(JsonRecord), + requestId: nullableText, + originIp: nullableText, + originCountry: nullableText, +}) + +export const decodeStoredAuditLogEntry = Schema.decodeUnknownEffect(StoredAuditLogEntry) + +/** A decoded warehouse row as an `AuditLogEntry`. */ +export const storedRowToEntry = ( + orgId: OrgId, + row: Schema.Schema.Type, +): AuditLogEntry => ({ + orgId, + ...row, + // An entry with no diff has no changed fields either; the row stores `[]`. + changedFields: row.changes === null ? null : row.changedFields, +}) diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 8db63c1ac..6079b6e27 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -4,6 +4,10 @@ import { Effect, Layer, Option, Schema } from "effect" import { ApiKeysService } from "@/services/org/ApiKeysService" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { recordApiDenial } from "@/services/auth/audit-denial" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) @@ -22,10 +26,11 @@ export const ApiAuthorizationLayer = Layer.effect( Effect.gen(function* () { const env = yield* Env const apiKeys = yield* ApiKeysService + const audit = yield* AuditLogService const resolveTenant = makeResolveTenant(env) return CurrentTenant.Authorization.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -42,12 +47,26 @@ export const ApiAuthorizationLayer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + // Denied attempts are audited with the same attribution as + // successes — a key probing a surface it is not valid for is + // exactly what the audit log exists to surface. This layer has + // no rate limiter, so coalescing is what bounds the volume. + const recordDenied = (denialReason: string) => + recordApiDenial(audit, request, { + orgId: resolved.orgId, + userId: resolved.userId, + apiKeyId: resolved.keyId, + apiKeyName: resolved.name, + denialReason, + }) if (resolved.kind !== "standard") { + yield* recordDenied("This API key is only valid for the MCP server") return yield* new UnauthorizedError({ message: "This API key is only valid for the MCP server", }) } if (resolved.scopes !== null) { + yield* recordDenied("Restricted API keys must use the /v2 API") return yield* new UnauthorizedError({ message: "Restricted API keys must use the /v2 API", }) @@ -63,15 +82,37 @@ export const ApiAuthorizationLayer = Layer.effect( roles: resolved.roles ?? apiKeyDefaultRoles, authMode: "self_hosted", }) - return yield* Effect.provideService(httpEffect, CurrentTenant.Context, tenant) + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, { + type: "api_key", + apiKeyId: resolved.keyId, + label: resolved.name, + source: "api", + }), + withAuditedRead(audit, request, options, { + orgId: resolved.orgId, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + label: resolved.name, + }, + source: "api", + }), + ) } const tenant = yield* resolveTenant(request.headers) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "dashboard", + }), ) }), }) diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index b4eb00acf..4192671c4 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -15,6 +15,10 @@ import { ORG_SELECTION_HEADER } from "@maple/auth" import { makeResolveTenant } from "./AuthService" import { OrgMembershipService } from "@/services/auth/OrgMembershipService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { recordApiDenial } from "@/services/auth/audit-denial" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" import { API_V2_RATE_LIMIT_PERIOD_SECONDS, @@ -67,6 +71,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( const env = yield* Env const apiKeys = yield* ApiKeysService const rateLimiter = yield* ApiV2RateLimiter + const audit = yield* AuditLogService // The one resolver wired for organization selection: `x-maple-org-id` is // a v2-client affordance (the iOS app publishing a widget snapshot per // organization), and every other resolver rejects the header instead. @@ -83,7 +88,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( ) return AuthorizationV2.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -92,6 +97,18 @@ export const ApiAuthorizationV2Layer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + // A refused attempt is the highest-signal audit row there is — + // denials carry the same actor attribution as successes, tagged + // `outcome: "denied"`, coalesced so a looping client cannot + // amplify into unbounded rows. + const recordDenied = (denialReason: string) => + recordApiDenial(audit, request, { + orgId: resolved.orgId, + userId: resolved.userId, + apiKeyId: resolved.keyId, + apiKeyName: resolved.name, + denialReason, + }) // Deny-list, not an allow-list: `mcp` keys are minted through a // path that does not gate on organization admin, so they must // never reach the public API. `device` keys are admitted @@ -99,9 +116,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // pinned roles below — is chosen by the server that minted // them, not by whatever is holding them. if (resolved.kind === "mcp") { - return yield* Effect.fail( - V2InvalidCredentials.make("This API key is only valid for the MCP server."), - ) + const message = "This API key is only valid for the MCP server." + yield* recordDenied(message) + return yield* Effect.fail(V2InvalidCredentials.make(message)) } // A device credential's authority is entirely its pinned @@ -110,9 +127,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // permissive default — it is a key whose defining property // is missing, so it is rejected rather than promoted. if (resolved.kind === "device" && resolved.roles === null) { - return yield* Effect.fail( - V2InvalidCredentials.make("This device credential is not valid."), - ) + const message = "This device credential is not valid." + yield* recordDenied(message) + return yield* Effect.fail(V2InvalidCredentials.make(message)) } // Attribute before the scope check so scope-rejected @@ -159,11 +176,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( ) } if (!scopeAllows(resolved.scopes, required)) { - return yield* Effect.fail( - V2InsufficientScope.make( - `This API key does not have the "${required.family}:${required.access}" scope required for this request.`, - ), - ) + const message = `This API key does not have the "${required.family}:${required.access}" scope required for this request.` + yield* recordDenied(message) + return yield* Effect.fail(V2InsufficientScope.make(message)) } // An API key is already organization-bound, so a selection could @@ -172,11 +187,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // check has to be here too. const requestedOrg = getOrgSelectionHeader(request.headers) if (requestedOrg !== undefined && requestedOrg !== resolved.orgId) { - return yield* Effect.fail( - V2OrganizationAccessDenied.make( - "An API key cannot select a different organization.", - ), - ) + const message = "An API key cannot select a different organization." + yield* recordDenied(message) + return yield* Effect.fail(V2OrganizationAccessDenied.make(message)) } const tenant = new CurrentTenant.TenantSchema({ @@ -186,7 +199,26 @@ export const ApiAuthorizationV2Layer = Layer.effect( authMode: "self_hosted", ...(resolved.scopes !== null ? { scopes: resolved.scopes } : undefined), }) - return yield* Effect.provideService(httpEffect, CurrentTenant.Context, tenant) + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, { + type: "api_key", + apiKeyId: resolved.keyId, + label: resolved.name, + source: "api", + }), + // Telemetry and replay reads are recorded (see `AuditedRead`). + withAuditedRead(audit, request, options, { + orgId: resolved.orgId, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + label: resolved.name, + }, + source: "api", + }), + ) } const tenant = yield* resolveTenant(request.headers).pipe( @@ -195,10 +227,14 @@ export const ApiAuthorizationV2Layer = Layer.effect( ), ) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "dashboard", + }), ) }), }) diff --git a/apps/api/src/services/auth/SessionAuthorizationLayer.ts b/apps/api/src/services/auth/SessionAuthorizationLayer.ts index 3917091d7..a2e77d3f2 100644 --- a/apps/api/src/services/auth/SessionAuthorizationLayer.ts +++ b/apps/api/src/services/auth/SessionAuthorizationLayer.ts @@ -4,6 +4,9 @@ import { CurrentTenant } from "@maple/domain/http" import { Effect, Layer } from "effect" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" const getBearerToken = (headers: Record): string | undefined => { @@ -31,10 +34,11 @@ export const SessionAuthorizationLayer = Layer.effect( CurrentTenant.SessionAuthorization, Effect.gen(function* () { const env = yield* Env + const audit = yield* AuditLogService const resolveTenant = makeResolveTenant(env) return CurrentTenant.SessionAuthorization.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -47,10 +51,16 @@ export const SessionAuthorizationLayer = Layer.effect( const tenant = yield* resolveTenant(request.headers) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + const actor = { type: "user", source: "dashboard" } as const + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, actor), + // Telemetry and replay reads are recorded (see `AuditedRead`). + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: actor.source, + }), ) }), }) diff --git a/apps/api/src/services/auth/audit-actor.ts b/apps/api/src/services/auth/audit-actor.ts new file mode 100644 index 000000000..bb917cb45 --- /dev/null +++ b/apps/api/src/services/auth/audit-actor.ts @@ -0,0 +1,37 @@ +import { Context } from "effect" +import type { AuditLogSource } from "@maple/domain/http" +import type { ApiKeyId } from "@maple/domain/primitives" + +/** + * How the current request authenticated, for audit attribution. The tenant + * context deliberately does not say whether a request came from a dashboard + * session, an API key, or MCP — this reference carries those two facts, which + * nothing downstream can re-derive. + */ +export interface AuditActorInfo { + /** `system` is Maple itself acting through an internal service token. */ + readonly type: "user" | "api_key" | "system" + readonly apiKeyId?: ApiKeyId + /** + * A display name for the credential, frozen into the entry. Set for API + * keys, whose name is already on the row auth resolved; a dashboard session + * has no name to carry (Clerk's claims hold none), so those entries are + * labelled when the log is read. + */ + readonly label?: string + /** The surface the request arrived through, recorded as the entry's `source`. */ + readonly source: AuditLogSource +} + +/** + * A reference (typed default, no handler requirement) rather than a service: + * the auth middlewares override it per request, and handlers that never record + * audit entries are unaffected. `undefined` means the request skipped the + * standard auth middlewares (internal tokens, queue consumers, crons) — callers + * must then fall back to whatever attribution they can establish themselves, + * never assume a dashboard session. + */ +export class CurrentAuditActor extends Context.Reference( + "@maple/api/services/auth/CurrentAuditActor", + { defaultValue: () => undefined }, +) {} diff --git a/apps/api/src/services/auth/audit-denial.ts b/apps/api/src/services/auth/audit-denial.ts new file mode 100644 index 000000000..428755efe --- /dev/null +++ b/apps/api/src/services/auth/audit-denial.ts @@ -0,0 +1,89 @@ +import { Clock, Effect } from "effect" +import type { HttpServerRequest } from "effect/unstable/http" +import type { ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import { httpRequestForensics, type AuditLogServiceApi } from "@/services/audit/AuditLogService" + +/** Suppress duplicate denial rows for the same key/reason within this window. */ +export const AUDIT_DENIAL_COALESCE_WINDOW_MS = 60_000 + +/** Bound on distinct in-flight denial signatures kept per isolate. */ +const MAX_TRACKED_DENIALS = 10_000 + +/** + * Isolate-local coalescing cache: last-recorded time per denial signature. + * Tradeoff: Workers isolates multiply and recycle, so suppression is + * best-effort — each isolate still records the first denial it sees, which is + * the forensic signal; only the repeat volume is shed, with no network hop. + */ +const recentDenials = new Map() + +/** Test-only: clear the isolate-local coalescing state between cases. */ +export const resetAuditDenialCoalescing = (): void => { + recentDenials.clear() +} + +/** + * True when this signature has not been recorded within the window; marks it + * recorded. The timestamp is not refreshed on suppression, so a sustained loop + * still lands one row per window rather than going silent forever. + */ +const shouldRecordDenial = (signature: string, now: number): boolean => { + const last = recentDenials.get(signature) + if (last !== undefined && now - last < AUDIT_DENIAL_COALESCE_WINDOW_MS) return false + // Delete-then-set keeps insertion order ≈ recency, so the bound evicts the stalest signature. + recentDenials.delete(signature) + if (recentDenials.size >= MAX_TRACKED_DENIALS) { + const oldest = recentDenials.keys().next() + if (!oldest.done) recentDenials.delete(oldest.value) + } + recentDenials.set(signature, now) + return true +} + +export interface ApiDenialInput { + readonly orgId: OrgId + readonly userId: UserId + readonly apiKeyId: ApiKeyId + /** The key's name, frozen onto the entry — a refused key is often revoked next. */ + readonly apiKeyName: string + readonly denialReason: string +} + +const requestPath = (url: string): string => { + const queryStart = url.indexOf("?") + return queryStart === -1 ? url : url.slice(0, queryStart) +} + +/** + * Record a denied public-API request with full forensics (method+path plus the + * `cf-ray`/`cf-connecting-ip`/`cf-ipcountry` headers), coalescing duplicates: + * the same (org, key, method+path, reason) is written at most once per window + * so a client looping mis-scoped requests cannot amplify into unbounded queue + * messages, rows, and warn logs. Never fails — same contract as `record`. + */ +export const recordApiDenial = ( + audit: AuditLogServiceApi, + request: HttpServerRequest.HttpServerRequest, + input: ApiDenialInput, +): Effect.Effect => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const path = requestPath(request.url) + const signature = `${input.orgId}|${input.apiKeyId}|${request.method} ${path}|${input.denialReason}` + if (!shouldRecordDenial(signature, now)) return + yield* audit.record({ + orgId: input.orgId, + actor: { + type: "api_key", + userId: input.userId, + apiKeyId: input.apiKeyId, + label: input.apiKeyName, + }, + source: "api", + action: "api.request", + outcome: "denied", + denialReason: input.denialReason, + metadata: { method: request.method, path }, + ...httpRequestForensics(request), + }) + }) diff --git a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts index 8e6120f40..2097be7f2 100644 --- a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts @@ -7,6 +7,7 @@ import { Clock, Effect, Layer, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" import { msToDate } from "@/platform/time" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { WarehouseQueryService, type WarehouseQueryServiceApi, @@ -66,7 +67,11 @@ const makeWarehouseStub = (contexts: Array): WarehouseQueryServiceApi => const makeLayer = (contexts: Array) => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) - const workflow = ErrorIssueWorkflowService.layer.pipe(Layer.provide(database), Layer.provide(actors)) + const workflow = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layerMemory), + Layer.provide(database), + Layer.provide(actors), + ) const warehouse = Layer.succeed(WarehouseQueryService, makeWarehouseStub(contexts)) const readModels = readRequirements.pipe( Layer.provide(database), diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts index 064531713..c8435e329 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts @@ -20,13 +20,17 @@ import type { MapleDatabaseTransaction } from "@maple/db/client" import { and, eq } from "drizzle-orm" import { Database, type DatabaseApi, type DatabaseClient } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "./ErrorActorsService" import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" // Compile-time guard: broadening this service to warehouse, cache, Env, // notifications, or WorkerEnvironment makes this assignment fail. -const databaseAndActorsOnly: Layer.Layer = - ErrorIssueWorkflowService.layer +const databaseAndActorsOnly: Layer.Layer< + ErrorIssueWorkflowService, + never, + Database | ErrorActorsService | AuditLogService +> = ErrorIssueWorkflowService.layer const asOrgId = Schema.decodeUnknownSync(OrgId) const asPullRequestId = Schema.decodeUnknownSync(ErrorIssuePullRequestId) @@ -44,7 +48,8 @@ afterEach(() => cleanupTestDbs(createdDbs)) const makeLayer = () => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) - const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(database, actors))) + const audit = AuditLogService.layerMemory + const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(database, actors, audit))) return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(database)) } @@ -94,7 +99,9 @@ const makeFaultyLayer = (failTable: unknown) => { }), ).pipe(Layer.provide(database)) const actors = ErrorActorsService.layer.pipe(Layer.provide(faulty)) - const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(faulty, actors))) + const workflow = databaseAndActorsOnly.pipe( + Layer.provide(Layer.mergeAll(faulty, actors, AuditLogService.layerMemory)), + ) return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(faulty)) } diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index 661b7a07d..2e431a723 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -23,6 +23,7 @@ import { MACHINE_OWNED_WORKFLOW_STATES, } from "@maple/domain/http" import { + actors, alertIncidents, errorIncidents, errorIssues, @@ -37,6 +38,9 @@ import { import { and, desc, eq, inArray, sql } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { SYSTEM_ERRORS_AGENT_NAME } from "@/services/auth/system-actors" import { readTxid, txidColumn } from "@/platform/electric-txid" import { dateToMs, msToDate } from "@/platform/time" import { ErrorActorsService } from "./ErrorActorsService" @@ -185,10 +189,15 @@ export interface ErrorIssueWorkflowServiceApi extends ErrorIssueWorkflowPublicAp > } -const make: Effect.Effect = Effect.gen( +const make: Effect.Effect< + ErrorIssueWorkflowServiceApi, + never, + Database | ErrorActorsService | AuditLogService +> = Effect.gen( function* () { const database = yield* Database - const actors = yield* ErrorActorsService + const actorsService = yield* ErrorActorsService + const audit = yield* AuditLogService const dbExecute = makeErrorDatabaseExecute(database, "ErrorIssueWorkflowService") const newEventId = () => decodeEventIdSync(randomUUID()) @@ -392,7 +401,7 @@ const make: Effect.Effect row.id) const openSet = yield* issuesWithOpenIncidents(orgId, issueIds) const activityMap = yield* issueActivityRollups(orgId, issueIds) - const actorMap = yield* actors.collectActorDocs( + const actorMap = yield* actorsService.collectActorDocs( orgId, rows.flatMap((row) => [row.assignedActorId ?? null, row.leaseHolderActorId ?? null]), ) @@ -431,12 +440,92 @@ const make: Effect.Effect + Effect.gen(function* () { + const rows = yield* dbExecute((db) => + db + .select() + .from(actors) + .where(and(eq(actors.orgId, orgId), eq(actors.id, actorId))) + .limit(1), + ) + const actor = rows[0] + if (actor === undefined || (actor.type !== "agent" && actor.type !== "user")) return + // Maple's own sweeps run as an agent actor (`ensureSystemActor` mints + // one), so without this check auto-close, lease expiry and fix + // verification all read as a third-party agent acting over MCP. + const isSystemActor = actor.type === "agent" && actor.agentName === SYSTEM_ERRORS_AGENT_NAME + // The actors row knows *who*, never *how*: it is the same row whether + // the mutation arrived from the dashboard, an API key, or MCP. The + // request's `CurrentAuditActor` is the only thing that knows the + // credential and surface, so a human actor is attributed through it and + // falls back to a dashboard session only when nothing set it (queue + // consumers, crons). + const request = yield* CurrentAuditActor + yield* audit.record({ + orgId, + actor: isSystemActor + ? { type: "system", actorId, label: SYSTEM_ERRORS_AGENT_NAME } + : actor.type === "agent" + ? { + type: "agent", + actorId, + ...(actor.agentName === null ? undefined : { label: actor.agentName }), + // On-behalf-of: the human who registered the agent, the + // closest authority the actor registry records. + ...(actor.createdBy === null ? undefined : { userId: actor.createdBy }), + } + : { + type: request?.type ?? "user", + ...(actor.userId === null ? undefined : { userId: actor.userId }), + ...(request?.apiKeyId === undefined + ? undefined + : { apiKeyId: request.apiKeyId }), + actorId, + }, + source: isSystemActor + ? "system" + : actor.type === "agent" + ? "mcp" + : (request?.source ?? "dashboard"), + action: `error_issue.${type}`, + resourceId: issueId, + metadata: { + ...(opts.fromState != null ? { from_state: opts.fromState } : undefined), + ...(opts.toState != null ? { to_state: opts.toState } : undefined), + }, + }) + }).pipe( + // Typed failures and defects only — an interrupt must propagate so + // fiber teardown never triggers a stray write. + Effect.catch((error) => Effect.logWarning("Issue event audit write failed", { issueId, cause: error })), + Effect.catchDefect((defect) => + Effect.logWarning("Issue event audit write failed", { issueId, cause: defect }), + ), + ) + const recordEvent: ErrorIssueWorkflowServiceApi["recordEvent"] = Effect.fn( "ErrorsService.recordEvent", )(function* (orgId, issueId, actorId, type, opts = {}) { const timestamp = opts.timestamp ?? (yield* Clock.currentTimeMillis) const insert = buildEventInsert(orgId, issueId, actorId ?? null, type, timestamp, opts) - return yield* dbExecute((db) => db.insert(errorIssueEvents).values(insert)) + const inserted = yield* dbExecute((db) => db.insert(errorIssueEvents).values(insert)) + // System/sweep events carry no actor and stay out of the audit log. + if (actorId !== null) { + yield* recordEventAudit(orgId, issueId, actorId, type, opts) + } + return inserted }) /** @@ -556,7 +645,10 @@ const make: Effect.Effect db.insert(errorIssueEvents).values(row)) - yield* actors.touchActor(orgId, actorId, timestamp) - const actorMap = yield* actors.collectActorDocs(orgId, [actorId]) + // This path writes the event row itself rather than going through + // `recordEvent`, so the audit mirror has to be invoked explicitly. The + // comment body stays out of the row — the audit records that a comment + // was made, not what it said. + yield* recordEventAudit(orgId, issueId, actorId, type, {}) + yield* actorsService.touchActor(orgId, actorId, timestamp) + const actorMap = yield* actorsService.collectActorDocs(orgId, [actorId]) return rowToEvent(row, actorMap) }) @@ -849,7 +949,7 @@ const make: Effect.Effect row.actorId ?? null), ) diff --git a/apps/api/src/services/errors/ErrorsService.test.ts b/apps/api/src/services/errors/ErrorsService.test.ts index d981ddddf..f1b0c87c1 100644 --- a/apps/api/src/services/errors/ErrorsService.test.ts +++ b/apps/api/src/services/errors/ErrorsService.test.ts @@ -39,6 +39,7 @@ import { Env } from "@/platform/Env" import { isRetryablePostgresContention } from "@/platform/postgres-errors" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { msToDate } from "@/platform/time" +import { AuditLogService } from "@/services/audit/AuditLogService" import type { SqlQueryOptions, WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { ErrorActorsService } from "./ErrorActorsService" @@ -211,6 +212,7 @@ const makeErrorsLayer = ( const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) @@ -300,6 +302,7 @@ const makeGatingLayer = (opts: { const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) diff --git a/apps/api/src/services/errors/IssueFixVerificationService.test.ts b/apps/api/src/services/errors/IssueFixVerificationService.test.ts index 278d1f629..c970a5aa0 100644 --- a/apps/api/src/services/errors/IssueFixVerificationService.test.ts +++ b/apps/api/src/services/errors/IssueFixVerificationService.test.ts @@ -9,6 +9,7 @@ import { eq } from "drizzle-orm" import { Database, type DatabaseApi, type DatabaseClient } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "./ErrorActorsService" import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" import { PullRequestLookup } from "./PullRequestLookup" @@ -68,6 +69,7 @@ const makeLayer = ( const envLive = Env.layer.pipe(Layer.provide(testConfig())) const actorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const workflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(actorsLive), ) diff --git a/apps/api/src/services/org/ApiKeysService.ts b/apps/api/src/services/org/ApiKeysService.ts index 673ca3097..0898a6285 100644 --- a/apps/api/src/services/org/ApiKeysService.ts +++ b/apps/api/src/services/org/ApiKeysService.ts @@ -26,6 +26,8 @@ export interface ResolvedApiKey { readonly orgId: OrgId readonly userId: UserId readonly keyId: ApiKeyId + /** The key's display name, frozen into audit entries at write time. */ + readonly name: string readonly kind: ApiKeyKind readonly metadataJson: string | null /** v2 scope strings; null = legacy full access. */ @@ -594,6 +596,7 @@ export class ApiKeysService extends Context.Service()("@maple/ap orgId: row.value.orgId, userId: row.value.createdBy, keyId: row.value.id, + name: row.value.name, kind: row.value.kind, metadataJson: row.value.metadataJson == null ? null : JSON.stringify(row.value.metadataJson), scopes: row.value.scopes ?? null, diff --git a/apps/api/src/services/org/OrgMembersService.ts b/apps/api/src/services/org/OrgMembersService.ts index 45e10c1b0..cb88b2492 100644 --- a/apps/api/src/services/org/OrgMembersService.ts +++ b/apps/api/src/services/org/OrgMembersService.ts @@ -14,6 +14,8 @@ export interface OrgMember { readonly userId: string readonly email: string readonly name: string | null + /** Provider-hosted avatar; Clerk serves one for every user, initials included. */ + readonly imageUrl: string | null } export interface OrgMembersServiceApi { @@ -22,6 +24,17 @@ export interface OrgMembersServiceApi { * Fails when any id is not a member of the org, or when member resolution * is unavailable (self-hosted mode without Clerk). */ + /** + * Every member of the org. Unlike {@link resolveMembers} this answers for + * the directory as it is now, so a caller labelling historical records gets + * the members it can name and nothing for ids that have since left. + */ + readonly listMembers: ( + orgId: OrgId, + ) => Effect.Effect< + ReadonlyArray, + AlertMemberDirectoryNotConfiguredError | AlertMemberDirectoryUnavailableError + > readonly resolveMembers: ( orgId: OrgId, userIds: ReadonlyArray, @@ -80,7 +93,7 @@ const make = Effect.gen(function* () { [member.publicUserData?.firstName, member.publicUserData?.lastName] .filter(Boolean) .join(" ") || null - all.push({ userId, email, name }) + all.push({ userId, email, name, imageUrl: member.publicUserData?.imageUrl ?? null }) } offset += page.data.length if (offset >= page.totalCount || page.data.length === 0) break @@ -119,7 +132,7 @@ const make = Effect.gen(function* () { return resolved }) - return { resolveMembers } satisfies OrgMembersServiceApi + return { listMembers, resolveMembers } satisfies OrgMembersServiceApi }) export class OrgMembersService extends Context.Service()( diff --git a/apps/api/src/services/product-events/clerk-events.ts b/apps/api/src/services/product-events/clerk-events.ts index 875c3635b..6e74ac6e8 100644 --- a/apps/api/src/services/product-events/clerk-events.ts +++ b/apps/api/src/services/product-events/clerk-events.ts @@ -30,7 +30,12 @@ export const ClerkUserCreatedData = Schema.Struct({ }) /** - * `organizationMembership.deleted` / `.updated`. + * `organizationMembership.*` payload. Membership is managed in Clerk directly — + * the web app never asks Maple's API to add or remove a member — so this + * webhook is both where those changes are audited and where the revocation + * sweep runs. Clerk does not name the admin who made the change, only the + * member it happened to, which is why the audit entries are attributed to + * `system`. * * `role` is the member's role *after* the change. It is trusted only to answer * "did this member just stop being an admin" — a question whose safe direction @@ -42,6 +47,21 @@ export const ClerkOrganizationMembershipData = Schema.Struct({ public_user_data: Schema.Struct({ user_id: Schema.String }), role: Schema.optionalKey(Schema.String), }) +export type ClerkOrganizationMembershipData = Schema.Schema.Type< + typeof ClerkOrganizationMembershipData +> + +/** The membership verbs Maple audits, keyed by Clerk's event type. */ +export const CLERK_MEMBERSHIP_EVENTS = { + "organizationMembership.created": "added", + "organizationMembership.updated": "role_changed", + "organizationMembership.deleted": "removed", +} as const satisfies Record + +export type ClerkMembershipEventType = keyof typeof CLERK_MEMBERSHIP_EVENTS + +export const isClerkMembershipEvent = (type: string): type is ClerkMembershipEventType => + Object.hasOwn(CLERK_MEMBERSHIP_EVENTS, type) /** * `user.deleted`. Clerk's `DeletedObjectJSON` declares `id` optional, so a diff --git a/apps/api/src/services/warehouse/warehouse-catalog.ts b/apps/api/src/services/warehouse/warehouse-catalog.ts index f97dd071c..2ec635840 100644 --- a/apps/api/src/services/warehouse/warehouse-catalog.ts +++ b/apps/api/src/services/warehouse/warehouse-catalog.ts @@ -99,12 +99,23 @@ export interface TableInfo extends TableSummary { readonly partitionKey?: string } +/** + * Datasources that raw SQL must never reach, even inside the caller's own org. + * The audit log records every member's activity and origin IP and is served + * only through the admin-gated `GET /v2/audit_log`; letting `run_sql` or a + * dashboard widget read it would bypass that gate (and let the log observe + * itself being read). + */ +const RAW_SQL_HIDDEN_DATASOURCES: ReadonlySet = new Set(["audit_log"]) + function collectDatasources() { // `Datasources` exports a mix of datasource definitions, type aliases, helper // functions, and constant lookup tables. `isDatasourceDefinition` is the // runtime filter; we cast to `unknown` first because the static union of all // exports is too wide for TS to narrow with the predicate. - return (Object.values(Datasources) as ReadonlyArray).filter(isDatasourceDefinition) + return (Object.values(Datasources) as ReadonlyArray) + .filter(isDatasourceDefinition) + .filter((ds) => !RAW_SQL_HIDDEN_DATASOURCES.has(ds._name)) } export function listWarehouseTables(): ReadonlyArray { diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index e5284815a..12435d911 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -3,7 +3,13 @@ import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" import { WorkerConfigProviderLayer, workerEnvironmentLayer } from "@maple/infra/worker-runtime" import { Cause, Effect, Layer, Option } from "effect" +import { EdgeCacheService } from "@maple/cache" +import { CacheBackendLive } from "@/platform/CacheBackendLive" import { layerPg } from "@/platform/DatabasePgLive" +import { TinybirdOrgTokenService } from "@/services/integrations/TinybirdOrgTokenService" +import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { AuditLogService } from "@/services/audit/AuditLogService" import { Env } from "@/platform/Env" import { GithubAppClient } from "./services/integrations/vcs/vendor/github/GithubAppClient" import { GithubHttp } from "./services/integrations/vcs/vendor/github/GithubHttp" @@ -57,8 +63,21 @@ export const buildVcsSyncLayer = (_env: Record) => { // the scheduled producer below never sees a PR event — so it is built here // rather than in `Base`, keeping the cron layer as light as it was. const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(Base)) + // Issue events from a PR webhook are audited, and audit entries are warehouse + // rows — so the consumer carries the (Tinybird-pinned) ingest path as well. + const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive)) + const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(Base, EdgeCacheServiceLive)), + ) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), + ) + const AuditLogServiceLive = AuditLogService.layer.pipe( + Layer.provide(Layer.mergeAll(WarehouseQueryServiceLive, workerEnvironmentLayer)), + ) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(Layer.mergeAll(Base, ErrorActorsServiceLive)), + Layer.provide(Layer.mergeAll(Base, ErrorActorsServiceLive, AuditLogServiceLive)), ) const IssueFixVerificationServiceLive = IssueFixVerificationService.layer.pipe( Layer.provide( diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index f24af09bc..5eba84afa 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -417,15 +417,24 @@ const handleQueue = async ( processPlanetScaleWebhookBatch, flushPlanetScaleWebhookTelemetry, } = await import("./planetscale-webhook-runtime") - try { - await runScheduledEffect( - buildPlanetScaleWebhookLayer(env), - await scoped(processPlanetScaleWebhookBatch(batch)), - ctx, - ) - } finally { - ctx.waitUntil(flushPlanetScaleWebhookTelemetry(env)) - } + await runScheduledEffect( + buildPlanetScaleWebhookLayer(env), + await scoped(processPlanetScaleWebhookBatch(batch)), + ctx, + { onSettled: () => flushPlanetScaleWebhookTelemetry(env) }, + ) + return + } + if (queueKind === "audit-events") { + const { buildAuditEventsLayer, processAuditEventsBatch, flushAuditEventsTelemetry } = await import( + "./audit-events-runtime" + ) + await runScheduledEffect( + buildAuditEventsLayer(env), + await scoped(processAuditEventsBatch(batch)), + ctx, + { onSettled: () => flushAuditEventsTelemetry(env) }, + ) return } if (queueKind === "unknown") { @@ -433,11 +442,9 @@ const handleQueue = async ( } const { buildVcsSyncLayer, processBatch, flushVcsTelemetry } = await import("./vcs-sync-runtime") - try { - await runScheduledEffect(buildVcsSyncLayer(env), await scoped(processBatch(batch)), ctx) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + await runScheduledEffect(buildVcsSyncLayer(env), await scoped(processBatch(batch)), ctx, { + onSettled: () => flushVcsTelemetry(env), + }) } // Cron handler. Three schedules (see `crons` in alchemy.run.ts), dispatched on @@ -464,50 +471,37 @@ const handleScheduled = async ( const { runScrapeCheckRetention } = await import("@/services/integrations/scrape-check-retention") const { runPlanetScaleEventRetention } = await import("@/services/integrations/planetscale-event-retention") - try { - // Both sweeps ride this one cron: each new cron string costs an entry in - // alchemy.run.ts and a branch here, and neither needs its own beat. - // Sequential, not concurrent — they share one Postgres socket for the - // whole tick, so running them concurrently would only queue on it. - await runScheduledEffect( - buildScrapeRetentionLayer(env), - await scoped(Effect.andThen(runScrapeCheckRetention, runPlanetScaleEventRetention)), - ctx, - { onInterrupt: "graceful" }, - ) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + // Both sweeps ride this one cron: each new cron string costs an entry in + // alchemy.run.ts and a branch here, and neither needs its own beat. + // Sequential, not concurrent — they share one Postgres socket for the + // whole tick, so running them concurrently would only queue on it. + await runScheduledEffect( + buildScrapeRetentionLayer(env), + await scoped(Effect.andThen(runScrapeCheckRetention, runPlanetScaleEventRetention)), + ctx, + { onInterrupt: "graceful", onSettled: () => flushVcsTelemetry(env) }, + ) return } if (event.cron === SLACK_RECONCILE_CRON) { const { buildSlackReconcileLayer, runSlackReconciliation, flushSlackTelemetry } = await import("./slack-reconcile-runtime") - try { - await runScheduledEffect( - buildSlackReconcileLayer(env), - await scoped(runSlackReconciliation), - ctx, - { onInterrupt: "graceful" }, - ) - } finally { - ctx.waitUntil(flushSlackTelemetry(env)) - } + await runScheduledEffect(buildSlackReconcileLayer(env), await scoped(runSlackReconciliation), ctx, { + onInterrupt: "graceful", + onSettled: () => flushSlackTelemetry(env), + }) return } const { buildVcsScheduledLayer, runScheduledSync, flushVcsTelemetry } = await import("./vcs-sync-runtime") - try { - // Graceful on interrupt: a teardown mid-cron is expected lifecycle, and the - // schedule reruns — only the queue consumer above must keep rejecting so an - // interrupted batch redelivers instead of acking. - await runScheduledEffect(buildVcsScheduledLayer(env), await scoped(runScheduledSync), ctx, { - onInterrupt: "graceful", - }) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + // Graceful on interrupt: a teardown mid-cron is expected lifecycle, and the + // schedule reruns — only the queue consumer above must keep rejecting so an + // interrupted batch redelivers instead of acking. + await runScheduledEffect(buildVcsScheduledLayer(env), await scoped(runScheduledSync), ctx, { + onInterrupt: "graceful", + onSettled: () => flushVcsTelemetry(env), + }) } /** diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index a233922bb..eaf526c2c 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -181,4 +181,17 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "f7d559f0db216379db02bc78c4e50f180f40588e124adf65665bf7eaaf556735", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // TODO(v17): what changed, whether any part is rewritten or any row + // moves, and what this edge does NOT backfill. + // + // projectRevision is carried forward deliberately — it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 17, + fingerprint: "b3800f55258f0ae3", + digest: "b3800f55258f0ae37a52bec6e4fe38be8fa9daebe3c912db2aa6885a4d73fa20", + manifestDigest: "f19b88567770ee1b67f77d5734de61adbfd3ba907ce8ae28ce65a4da4e544533", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index f11aae74c..b36f2b4e8 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 16 as const +export const LOCAL_SCHEMA_VERSION = 17 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index afeaee76a..031deb60a 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -52,6 +52,7 @@ import { v12ToV13ServiceOperationsDiscriminatorsModule } from "./local-store-mig import { v13ToV14AiTraceIndexModule } from "./local-store-migrations/v13-to-v14-ai-trace-index" import { v14ToV15CommitShaVcsRevisionModule } from "./local-store-migrations/v14-to-v15-commit-sha-vcs-revision" import { v15ToV16AiTraceIndexFilterColumnsModule } from "./local-store-migrations/v15-to-v16-ai-trace-index-filter-columns" +import { v16ToV17AuditLogModule } from "./local-store-migrations/v16-to-v17-audit-log" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -125,6 +126,7 @@ export const localStoreMigrations: ReadonlyArray = v13ToV14AiTraceIndexModule, v14ToV15CommitShaVcsRevisionModule, v15ToV16AiTraceIndexFilterColumnsModule, + v16ToV17AuditLogModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v16-to-v17-audit-log.ts b/apps/cli/src/server/local-store-migrations/v16-to-v17-audit-log.ts new file mode 100644 index 000000000..089b6d616 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v16-to-v17-audit-log.ts @@ -0,0 +1,233 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cloneStoreForStaging } from "./journal-codecs" +import { resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V16, + LOCAL_SCHEMA_V16_MANIFEST, + LOCAL_SCHEMA_V16_SQL, + LOCAL_SCHEMA_V17, + LOCAL_SCHEMA_V17_MANIFEST, + LOCAL_SCHEMA_V17_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +const MODULE_ID = "local-0016-to-0017-audit-log" as const + +/** + * The local mirror of ClickHouse migration 0027. + * + * Purely additive: v17 introduces the `audit_log` table and touches nothing + * else — no existing table is read, rewritten or dropped, no view is replaced, + * and no row moves. Bootstrapping the v17 DDL over the cloned v16 store creates + * it through `CREATE TABLE IF NOT EXISTS`; every other statement is a no-op + * against objects that already exist. + * + * Nothing is backfilled, and there is nothing to backfill: local mode has no + * authenticated actors, so the table starts and stays empty here. It exists so + * a local store keeps mirroring the deployed schema. + * + * Every statement is idempotent, so a resume after a crash lands in the same + * place. + */ + +interface V16ToV17State { + readonly module: typeof MODULE_ID + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V16ToV17Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const isCount = (value: unknown): value is string => typeof value === "string" && /^\d+$/.test(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v16 -> v17 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (!isCount(count)) throw new Error(`v16 -> v17 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v16 -> v17 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V16ToV17State => { + if (!isRecord(value)) throw new Error("v16 -> v17 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v16 -> v17 state contains an unknown field") + if (value.module !== MODULE_ID || value.version !== 1) + throw new Error("v16 -> v17 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v16 -> v17 retentionDays must be an integer") + return { + module: MODULE_ID, + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V16ToV17Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v16 -> v17 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V16_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V16_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V16_SQL, bootstrapSchema: false }, + ) + return { + module: MODULE_ID, + version: 1, + rawRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async ( + context: MigrationModuleContext, + state: V16ToV17State, +): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await cloneStoreForStaging(source, target) + } + return state +} + +const apply = async (context: MigrationModuleContext): Promise => + // The v17 bootstrap creates `audit_log`; every other object already exists + // and its `IF NOT EXISTS` is a no-op. + context.openTarget(() => ({ installed: true }) as const, { + schemaSql: LOCAL_SCHEMA_V17_SQL, + bootstrapSchema: true, + }) + +const verify = async ( + context: MigrationModuleContext, + state: V16ToV17State, + _progress: V16ToV17Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V17_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v16 -> v17 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V17_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v16-store", + description: "Clone the stopped v16 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "create-audit-log", + description: "Create the empty audit_log table by bootstrapping the v17 schema", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v17-schema", + description: "Verify the v17 physical schema and the retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v16 store is cloned byte-for-byte before the new table is created.", + }, + { + name: "audit_log", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "Created empty; no existing table is read, rewritten, or dropped.", + }, +] + +export const v16ToV17AuditLogModule: LocalStoreMigrationModule< + V16ToV17State, + V16ToV17Progress +> = { + id: MODULE_ID, + moduleVersion: 1, + description: "Add the audit_log table", + from: LOCAL_SCHEMA_V16, + to: LOCAL_SCHEMA_V17, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 5c413fadd..c1c805916 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -15,6 +15,7 @@ import schemaV13Sql from "./schema/local-schema-v13.sql" with { type: "text" } import schemaV14Sql from "./schema/local-schema-v14.sql" with { type: "text" } import schemaV15Sql from "./schema/local-schema-v15.sql" with { type: "text" } import schemaV16Sql from "./schema/local-schema-v16.sql" with { type: "text" } +import schemaV17Sql from "./schema/local-schema-v17.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -75,6 +76,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV14Sql, schemaV15Sql, schemaV16Sql, + schemaV17Sql, ] export interface LocalSchemaSnapshot { @@ -131,6 +133,8 @@ export const LOCAL_SCHEMA_V15_SQL = snapshotAt(15).sql export const LOCAL_SCHEMA_V15_MANIFEST = snapshotAt(15).manifest export const LOCAL_SCHEMA_V16_SQL = snapshotAt(16).sql export const LOCAL_SCHEMA_V16_MANIFEST = snapshotAt(16).manifest +export const LOCAL_SCHEMA_V17_SQL = snapshotAt(17).sql +export const LOCAL_SCHEMA_V17_MANIFEST = snapshotAt(17).manifest export interface LocalSchemaIdentity { readonly version: number @@ -177,6 +181,7 @@ export const LOCAL_SCHEMA_V13 = identityAt(13) export const LOCAL_SCHEMA_V14 = identityAt(14) export const LOCAL_SCHEMA_V15 = identityAt(15) export const LOCAL_SCHEMA_V16 = identityAt(16) +export const LOCAL_SCHEMA_V17 = identityAt(17) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 8d1cc5ab2..9d566f057 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "fab3e18c388b21aa3ed50bdda5bcd00f0458a4e52a9c7b0cf00dfd0ac3f0b17b", + "projectRevision": "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v17.sql b/apps/cli/src/server/schema/local-schema-v17.sql new file mode 100644 index 000000000..1d7226d8f --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v17.sql @@ -0,0 +1,1960 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 49d71eba3dac82c8ce787ae477e2c314114b2dc907b2382265c4bc15aa0b86ca +-- localSchemaVersion: 16 + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) AS Model, + coalesce(nullIf(SpanAttributes['gen_ai.agent.name'], ''), SpanAttributes['ai.telemetry.functionId']) AS AgentName, + coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(((StatusCode = 'Error' OR SpanAttributes['error.type'] != '') OR SpanAttributes['gen_ai.response.status'] IN ('failed', 'error'))) AS IsError, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('chat', 'generate_content', 'text_completion', 'fetch_response') OR (((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND NOT ((coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%'))) AND NOT ((lower(SpanName) LIKE '%agent%' OR lower(SpanName) LIKE '%workflow%'))) AND (coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) != '' OR (lower(SpanName) LIKE '%chat%' OR lower(SpanName) LIKE '%completion%'))))) AS IsLlmCall, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('execute_tool') OR (coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND (coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%')))) AS IsToolCall, + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])) AS Tokens, + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cost'], ''), nullIf(SpanAttributes['gen_ai.usage.total_cost'], ''), SpanAttributes['llm.cost.total'])) AS Cost + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 7693085f0..61f1640b7 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: fab3e18c388b21aa3ed50bdda5bcd00f0458a4e52a9c7b0cf00dfd0ac3f0b17b --- localSchemaVersion: 16 +-- projectRevision: 9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced +-- localSchemaVersion: 17 CREATE TABLE IF NOT EXISTS ai_trace_index ( OrgId LowCardinality(String), @@ -80,6 +80,35 @@ PARTITION BY toDate(Hour) ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) TTL Hour + INTERVAL 90 DAY; +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + CREATE TABLE IF NOT EXISTS error_events ( OrgId LowCardinality(String), Timestamp DateTime, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index e61ff6e02..d1ffa1ba0 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -31,6 +31,7 @@ import { LOCAL_SCHEMA_V14, LOCAL_SCHEMA_V15, LOCAL_SCHEMA_V16, + LOCAL_SCHEMA_V17, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -78,16 +79,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v16 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("d975e674ce66af41") - expect(SCHEMA_DIGEST).toBe("d975e674ce66af417e4398d8ca336d41340c9c1aa7b26082a6c55ecd02effe38") + it("matches the generated v17 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("b3800f55258f0ae3") + expect(SCHEMA_DIGEST).toBe("b3800f55258f0ae37a52bec6e4fe38be8fa9daebe3c912db2aa6885a4d73fa20") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(16) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V16) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(17) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V17) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -155,7 +156,8 @@ describe("current local schema identity", () => { // bodies, so their object set is identical to v5 and the manifest digest // differs solely through those definitions. v9 removes `error_spans` and // its view; v11 replaces `web_events` with `product_events` and adds - // `identity_links`. Asserted as an exact set difference rather than a + // `identity_links`; v17 adds `audit_log`, which local mode creates but + // never writes. Asserted as an exact set difference rather than a // relaxed check, so a future edge still cannot add or drop an object // unnoticed. expect([...v5Names].filter((name) => !currentNames.has(name))).toEqual([ @@ -167,6 +169,7 @@ describe("current local schema identity", () => { expect([...currentNames].filter((name) => !v5Names.has(name))).toEqual([ "ai_trace_index", "ai_trace_index_mv", + "audit_log", "identity_links", "identity_links_mv", "product_events", @@ -255,7 +258,8 @@ describe("current local schema identity", () => { // v12 replaces two view bodies and v13 adds columns to two rollups; neither // adds an object. v14 is exactly the GenAI span index and its view, created - // empty and filled forward. + // empty and filled forward; v17 adds `audit_log`, created empty and never + // written in local mode. const v12Names = new Set(LOCAL_SCHEMA_V12_MANIFEST.objects.map((object) => object.name)) const v13Names = new Set(LOCAL_SCHEMA_V13_MANIFEST.objects.map((object) => object.name)) const currentSchemaNames = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) @@ -265,6 +269,7 @@ describe("current local schema identity", () => { expect([...currentSchemaNames].filter((name) => !v13Names.has(name))).toEqual([ "ai_trace_index", "ai_trace_index_mv", + "audit_log", ]) expect([...v13Names].filter((name) => !currentSchemaNames.has(name))).toEqual([]) const aiTraceIndex = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "ai_trace_index") @@ -312,6 +317,7 @@ describe("local migration registry", () => { "local-0013-to-0014-ai-trace-index", "local-0014-to-0015-commit-sha-vcs-revision", "local-0015-to-0016-ai-trace-index-filter-columns", + "local-0016-to-0017-audit-log", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -358,7 +364,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 17, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 18, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1363,6 +1369,7 @@ describe("v10 -> v11 product events module", () => { "local-0013-to-0014-ai-trace-index", "local-0014-to-0015-commit-sha-vcs-revision", "local-0015-to-0016-ai-trace-index-filter-columns", + "local-0016-to-0017-audit-log", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 5d3ddf426..f41a77989 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 16 and .schema == "d975e674ce66af41"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 17 and .schema == "b3800f55258f0ae3"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 30a0723e7..a79cb503b 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "fab3e18c388b21aa3ed50bdda5bcd00f0458a4e52a9c7b0cf00dfd0ac3f0b17b"; +pub const PROJECT_REVISION: &str = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index 7b31b3aa8..c68b2076c 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7335,6 +7335,10 @@ "description": "Ingest-time attribute rewrite rules. Move or copy span/resource attribute values to new keys as telemetry arrives, normalizing naming across services without redeploying them.", "name": "Attribute Mappings" }, + { + "description": "The organization's append-only audit trail — allowed and denied actions performed through the dashboard, the public API, and MCP, attributed to the user, API key, or agent that performed them, with before/after diffs for updates. Reading it requires organization-administrator access (or the `audit_log:read` scope for API keys).", + "name": "Audit Log" + }, { "description": "Metrics endpoints Maple scrapes on a schedule — self-hosted Prometheus endpoints and PlanetScale branch metrics. Manage targets, probe them on demand, and inspect recent scrape checks. Credentials are write-only.", "name": "Scrape Targets" diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx new file mode 100644 index 000000000..da8eb40b9 --- /dev/null +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -0,0 +1,736 @@ +import type { AuditActorType, AuditOutcome } from "@maple/domain/http" +import { + encodePublicId, + PublicIdPrefixes, + type V2AuditChanges, + type V2AuditLogEntry, +} from "@maple/domain/http/v2" +import { Option } from "effect" +import { useState, type ReactNode } from "react" + +import { Result, useAtomRefresh, useAtomValue } from "@/lib/effect-atom" +import { auditLogPageAtom } from "@/lib/services/atoms/audit-log-atoms" + +import { Avatar, AvatarFallback, AvatarImage } from "@maple/ui/components/ui/avatar" +import { Badge } from "@maple/ui/components/ui/badge" +import { Button } from "@maple/ui/components/ui/button" +import { CopyButton } from "@maple/ui/components/ui/copy-button" +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { trySync } from "@maple/ui/lib/try-sync" +import { cn } from "@maple/ui/lib/utils" +import { formatRelativeTime } from "@maple/ui/lib/time-format" +import { AlertWarningIcon, ArrowPathIcon, ChevronRightIcon, HistoryIcon } from "@/components/icons" + +type ActorFilter = AuditActorType | "all" +type OutcomeFilter = AuditOutcome | "all" + +const ACTOR_FILTERS: ReadonlyArray<{ value: ActorFilter; label: string }> = [ + { value: "all", label: "All" }, + { value: "user", label: "Users" }, + { value: "api_key", label: "API keys" }, + { value: "agent", label: "Agents" }, + { value: "system", label: "System" }, +] + +const OUTCOME_FILTERS: ReadonlyArray<{ value: OutcomeFilter; label: string }> = [ + { value: "all", label: "All" }, + { value: "allowed", label: "Allowed" }, + { value: "denied", label: "Denied" }, +] + +const ACTOR_BADGES: Record = { + user: { label: "User", variant: "secondary" }, + api_key: { label: "API key", variant: "success" }, + agent: { label: "Agent", variant: "info" }, + system: { label: "System", variant: "outline" }, +} satisfies Record + +// Shared column lanes so the header row and entry rows stay aligned. Resource and +// source collapse when the card is narrow; time + actor + action always stay +// visible, and action is the lane that absorbs the remaining width. The card, +// not the viewport, is what the breakpoints measure: the sidebar and settings +// nav leave it far narrower than the window. +// Below `@md` a row wraps: time + actor on the first line, the action on its own +// line beneath, indented past the chevron. +const COL = { + time: "flex w-[112px] shrink-0 items-center gap-2", + actor: "min-w-0 flex-1 @md:w-[176px] @md:flex-none", + action: "min-w-0 basis-full pl-5 @md:basis-0 @md:flex-1 @md:pl-0", + resource: "hidden w-[200px] min-w-0 shrink-0 @3xl:block", + // 52rem: the card's width at a 1440px window, the narrowest desktop that should still show it. + source: "hidden w-[104px] shrink-0 @min-[52rem]:block", +} +const COL_HEADER = "text-muted-foreground/70 font-mono text-[10px] uppercase tracking-[0.12em]" + +/** + * Up to two initials from a display name, for the moment before the avatar + * loads and for the members Clerk serves no picture for. An email falls back to + * its first letter rather than parsing a local part that is rarely a name. + */ +function initialsOf(label: string): string { + // A row we could not name falls back to the raw `user_…` id; "U" would read + // as a name it is not. + if (label.startsWith("user_")) return "?" + const words = label.trim().split(/\s+/).filter(Boolean) + if (words.length === 0 || label.includes("@")) return label.slice(0, 1).toUpperCase() + return words + .slice(0, 2) + .map((word) => word.slice(0, 1).toUpperCase()) + .join("") +} + +/** + * `user_3BfcmIS3bUNV6BfAEkR2WzFOCvu` → `user_…FOCvu`. The prefix says what kind + * of id it is and the tail is what someone compares against a copied value; + * the middle is noise in a 200px lane. The full id stays in the detail panel. + */ +function abbreviateId(id: string): string { + const prefixEnd = id.indexOf("_") + if (prefixEnd === -1 || id.length <= prefixEnd + 10) return id + return `${id.slice(0, prefixEnd + 1)}…${id.slice(-5)}` +} + +function actorDisplayName(entry: V2AuditLogEntry): string | null { + if (entry.actor_name !== null) return entry.actor_name + if (entry.actor_id !== null) return abbreviateId(entry.actor_id) + return null +} + +function formatDateTime(value: string): string { + return new Date(value).toLocaleString(undefined, { + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }) +} + +function formatDateTimeFull(value: string): string { + return new Date(value).toLocaleString(undefined, { + year: "numeric", + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + timeZoneName: "short", + }) +} + +/** `` / `` are the audit pipeline's placeholders, not values. */ +function isPlaceholder(value: unknown): value is string { + return typeof value === "string" && /^<[a-z]+>$/.test(value) +} + +// JSON.stringify(undefined) is undefined — surface it as text. +function formatScalar(value: unknown): string { + if (typeof value === "string") return value + return JSON.stringify(value) ?? "undefined" +} + +/** What a metadata string that opens with `{` or `[` parses to, when it parses at all. */ +type JsonDocument = Record | ReadonlyArray + +/** + * Metadata values are stored as they were recorded: request bodies and tool + * parameters arrive as JSON text, SQL as a statement. Anything structured, or + * long enough to wrap, is rendered as a block rather than inline. + */ +function metadataBlock(value: unknown): string | null { + if (typeof value === "string") { + const trimmed = value.trim() + const parsed = + trimmed.startsWith("{") || trimmed.startsWith("[") + ? Option.getOrNull(trySync((): JsonDocument => JSON.parse(trimmed))) + : null + if (parsed !== null) return JSON.stringify(parsed, null, 2) + return value.length > 72 || value.includes("\n") ? value : null + } + if (value !== null && typeof value === "object") return JSON.stringify(value, null, 2) + return null +} + +/** + * Append the next page, dropping any entry already shown. The pinned `until` + * ceiling makes overlap rare, but a filter re-fetch or a refresh mid-scroll can + * still repeat one — and a duplicated React key corrupts the list either way. + */ +function dedupeById( + existing: ReadonlyArray, + next: ReadonlyArray, +): V2AuditLogEntry[] { + const seen = new Set(existing.map((entry) => entry.id)) + return [...existing, ...next.filter((entry) => !seen.has(entry.id))] +} + +interface AuditLogView { + source: { data: ReadonlyArray } + entries: V2AuditLogEntry[] + hasMore: boolean + nextCursor: string | null +} + +export function AuditLogSection() { + const [actorFilter, setActorFilter] = useState("all") + const [outcomeFilter, setOutcomeFilter] = useState("all") + const [cursor, setCursor] = useState(undefined) + // Frozen on the first Load more, and cleared whenever the list restarts. The + // log is append-only and paginated by offset, so entries written mid-scroll + // would otherwise shift later pages and make them repeat and skip rows. + const [until, setUntil] = useState(undefined) + + const filterInput = { + ...(actorFilter !== "all" ? { actorType: actorFilter } : undefined), + ...(outcomeFilter !== "all" ? { outcome: outcomeFilter } : undefined), + } + const pageAtom = auditLogPageAtom({ + ...filterInput, + ...(cursor !== undefined ? { cursor } : undefined), + ...(until !== undefined ? { until } : undefined), + }) + // The first page for the current filters — what Refresh re-fetches. Without + // the refresh the page family would hand back its cached copy from before. + const firstPageAtom = auditLogPageAtom(filterInput) + const pageResult = useAtomValue(pageAtom) + const refreshPage = useAtomRefresh(pageAtom) + const refreshFirstPage = useAtomRefresh(firstPageAtom) + + // Each Load more / filter change swaps to a new page atom, which starts in its + // initial state. Keep the accumulated entries so the table stays rendered + // (dimmed) while the next page loads; a fresh (cursor-less) page replaces them. + const [view, setView] = useState(null) + if (Result.isSuccess(pageResult) && view?.source !== pageResult.value) { + setView({ + source: pageResult.value, + entries: + cursor === undefined + ? [...pageResult.value.data] + : dedupeById(view?.entries ?? [], pageResult.value.data), + hasMore: pageResult.value.has_more, + nextCursor: pageResult.value.next_cursor, + }) + } + + function restartList() { + setCursor(undefined) + setUntil(undefined) + } + + function handleFilterSelect(value: ActorFilter) { + if (value === actorFilter) return + setActorFilter(value) + restartList() + } + + function handleOutcomeSelect(value: OutcomeFilter) { + if (value === outcomeFilter) return + setOutcomeFilter(value) + restartList() + } + + function clearFilters() { + setActorFilter("all") + setOutcomeFilter("all") + restartList() + } + + function handleRefresh() { + restartList() + refreshFirstPage() + } + + const waiting = !Result.isSuccess(pageResult) || pageResult.waiting + const filtered = actorFilter !== "all" || outcomeFilter !== "all" + + return ( +
+

+ Changes, refused attempts, and every read of telemetry or session replays — from the dashboard, + API, and MCP. Select an entry for its full record. +

+ +
+
+ {ACTOR_FILTERS.map((filter) => ( + handleFilterSelect(filter.value)} + > + {filter.label} + + ))} +
+
+ {OUTCOME_FILTERS.map((filter) => ( + handleOutcomeSelect(filter.value)} + > + {filter.label} + + ))} +
+
+ +
+ +
+ {view === null && Result.isFailure(pageResult) ? ( + + + + + + Couldn't load the audit log + + Something went wrong while loading audit log entries. + + + + + ) : view === null ? ( +
+ + + +
+ ) : view.entries.length === 0 && filtered ? ( + + + + + + No entries match these filters + + Nothing recorded for this actor type and outcome. + + + + + ) : view.entries.length === 0 ? ( + + + + + + No audit log entries + + Actions and data reads by users, API keys, and agents will appear here. + + + + ) : ( +
+ + {view.entries.map((entry) => ( + + ))} +
+ )} +
+ + {view !== null && view.hasMore && view.nextCursor !== null && ( +
+ Showing {view.entries.length} entries — more available + +
+ )} +
+ ) +} + +function FilterTab({ + active, + onClick, + children, +}: { + active: boolean + onClick: () => void + children: ReactNode +}) { + return ( + + ) +} + +function ActorCell({ entry }: { entry: V2AuditLogEntry }) { + const badge = ACTOR_BADGES[entry.actor_type] + const name = actorDisplayName(entry) + + return ( +
+ {/* A person is shown as a face, not as the word "User" — the avatar + already says which kind of actor this is, and the name says who. + Keys, agents and system entries have no face and keep their badge. + The face is never lent to a key or an agent: those rows carry the + minting user's id too, and wearing it would credit a human for + something they may not have done. */} + {entry.actor_type === "user" ? ( + + {entry.actor_avatar_url !== null && } + {initialsOf(entry.actor_name ?? entry.actor_id ?? "")} + + ) : ( + + {badge.label} + + )} + {name !== null && ( + + {name} + + )} +
+ ) +} + +/** `alert_rule.updated` with the verb carrying the weight — it is what a scan of the column is for. */ +function ActionLabel({ action }: { action: string }) { + const dot = action.indexOf(".") + if (dot === -1) return {action} + return ( + + {action.slice(0, dot + 1)} + {action.slice(dot + 1)} + + ) +} + +function AuditLogRow({ entry }: { entry: V2AuditLogEntry }) { + const [expanded, setExpanded] = useState(false) + const detailId = `audit-entry-${entry.id}` + const denied = entry.outcome === "denied" + + return ( +
+ + {expanded && ( +
+ +
+ )} +
+ ) +} + +function ResourceCell({ entry }: { entry: V2AuditLogEntry }) { + // Membership changes act on a person: the affected user is the resource. + const id = entry.resource_id ?? entry.affected_user + if (entry.resource_type === null && id === null) { + return — + } + return ( +
+ {entry.resource_type !== null && ( + + {entry.resource_type} + + )} + {id !== null && ( + + {id} + + )} +
+ ) +} + +function DetailField({ label, children }: { label: string; children: ReactNode }) { + return ( + <> +
{label}
+
{children}
+ + ) +} + +/** An identifier with its copy affordance; the one place the full value is shown untruncated. */ +function Identifier({ value, label }: { value: string; label: string }) { + return ( + + {value} + + + ) +} + +function ChangeValue({ value }: { value: unknown }) { + if (isPlaceholder(value)) { + return {value.slice(1, -1)} + } + if (value === undefined || value === null || value === "") { + return — + } + return {formatScalar(value)} +} + +function ChangesTable({ changes }: { changes: V2AuditChanges }) { + return ( +
+ + + + + + + + + + {changes.fields.map((field) => ( + + + + + + ))} + +
FieldBeforeAfter
{field} + + + +
+
+ ) +} + +function MetadataList({ metadata }: { metadata: Record }) { + const keys = Object.keys(metadata) + if (keys.length === 0) return null + return ( +
+ {keys.map((key) => { + const value = metadata[key] + const block = metadataBlock(value) + return ( + + {block !== null ? ( + 
+								{block}
+
+ ) : ( + + + + )} + + ) + })} +
+ ) +} + +function AuditLogDetail({ entry }: { entry: V2AuditLogEntry }) { + const badge = ACTOR_BADGES[entry.actor_type] + const hasChanges = entry.changes !== null && entry.changes.fields.length > 0 + const hasMetadata = entry.metadata !== null && Object.keys(entry.metadata).length > 0 + + return ( +
+
+ + {formatDateTimeFull(entry.occurred_at)} + + + {formatDateTimeFull(entry.recorded_at)} + + + + + {badge.label} + + {entry.actor_name !== null && {entry.actor_name}} + {entry.actor_id !== null && } + + + + {entry.source} + {(entry.origin_ip !== null || entry.origin_country !== null) && ( + + {" · "} + {entry.origin_ip ?? "unknown IP"} + {entry.origin_country !== null && ` (${entry.origin_country})`} + + )} + + + + {entry.action} + {entry.outcome === "denied" ? ( + + Denied + + ) : ( + + Allowed + + )} + + {entry.denial_reason !== null && ( +

{entry.denial_reason}

+ )} + + + {entry.resource_type === null && entry.resource_id === null ? ( + — + ) : ( + + {entry.resource_type !== null && ( + + {entry.resource_type} + + )} + {entry.resource_id !== null && ( + + )} + + )} + + {entry.affected_user !== null && ( + + + + )} + {entry.request_id !== null && ( + + + + )} + + {/* The wire codec hands the client the raw id; show the `alog_…` + form the API itself returns, so it can be quoted back to it. */} + + +
+ + {hasChanges && entry.changes !== null && ( +
+

Changes

+ +
+ )} + + {hasMetadata && entry.metadata !== null && ( +
+

Details

+ +
+ )} +
+ ) +} diff --git a/apps/web/src/components/settings/settings-nav.tsx b/apps/web/src/components/settings/settings-nav.tsx index 1efc11c9b..db51b7bf2 100644 --- a/apps/web/src/components/settings/settings-nav.tsx +++ b/apps/web/src/components/settings/settings-nav.tsx @@ -14,6 +14,7 @@ import { DatabaseIcon, GearIcon, GridIcon, + HistoryIcon, KeyIcon, ServerIcon, ShieldIcon, @@ -26,6 +27,7 @@ import { SettingsNavShell } from "@/components/settings/settings-nav-shell" export const settingsTabValues = [ "organization", "members", + "audit-log", "setup-audit", "ingestion", "api-keys", @@ -41,6 +43,7 @@ export type SettingsTab = (typeof settingsTabValues)[number] export const settingsTabLabels: Record = { organization: "Organization", members: "Members", + "audit-log": "Audit Log", "setup-audit": "Setup Audit", ingestion: "Ingestion", "api-keys": "API Keys", @@ -108,6 +111,7 @@ const navSections: SettingsNavSection[] = [ items: [ { id: "organization", label: "Organization", icon: GearIcon }, { id: "members", label: "Members", icon: UserIcon }, + { id: "audit-log", label: "Audit Log", icon: HistoryIcon }, // Spans alerting, ingestion and integrations, so it sits at workspace level rather than // under any one of them. { id: "setup-audit", label: "Setup Audit", icon: CircleCheckIcon }, @@ -194,6 +198,9 @@ export function useVisibleSettingsSections() { ...section, items: section.items.filter((item) => { if (item.id === "data-platform") return canAccessDataPlatform + // `GET /v2/audit_log` is admin-only; hide the tab rather than let a + // member open it into a 403. + if (item.id === "audit-log") return isAdmin return true }), })) diff --git a/apps/web/src/lib/services/atoms/audit-log-atoms.ts b/apps/web/src/lib/services/atoms/audit-log-atoms.ts new file mode 100644 index 000000000..1e1ccf353 --- /dev/null +++ b/apps/web/src/lib/services/atoms/audit-log-atoms.ts @@ -0,0 +1,51 @@ +import type { AuditActorType, AuditOutcome } from "@maple/domain/http" +import { Effect } from "effect" +import { Atom } from "@/lib/effect-atom" +import { MapleApiV2AtomClient } from "@/lib/services/common/v2-atom-client" + +export const AUDIT_LOG_PAGE_LIMIT = 50 + +const ACTOR_TYPES: ReadonlyArray = ["user", "api_key", "agent", "system"] +const OUTCOMES: ReadonlyArray = ["allowed", "denied"] + +export interface AuditLogPageInput { + readonly cursor?: string + readonly actorType?: AuditActorType + readonly outcome?: AuditOutcome + /** + * Upper bound on `occurred_at`, pinned by the caller when it takes the first + * page. Pagination here is offset-based over a newest-first, append-only + * table, so an entry written mid-scroll shifts every later row down by one: + * without a frozen ceiling the next page repeats a row and skips another. + */ + readonly until?: string +} + +// Actor types and outcomes never contain "|", nor does an ISO timestamp, and the +// cursor is the trailing segment — so splitting on the first three separators +// stays unambiguous even for exotic cursors. +const family = Atom.family((key: string) => { + const [actorRaw = "", outcomeRaw = "", until = ""] = key.split("|", 3) + const cursor = key.slice(actorRaw.length + outcomeRaw.length + until.length + 3) + const actorType = ACTOR_TYPES.find((type) => type === actorRaw) + const outcome = OUTCOMES.find((value) => value === outcomeRaw) + + return MapleApiV2AtomClient.runtime.atom( + Effect.gen(function* () { + const client = yield* MapleApiV2AtomClient + return yield* client.auditLog.list({ + query: { + limit: AUDIT_LOG_PAGE_LIMIT, + ...(cursor !== "" ? { cursor } : undefined), + ...(actorType !== undefined ? { actor_type: actorType } : undefined), + ...(outcome !== undefined ? { outcome } : undefined), + ...(until !== "" ? { until } : undefined), + }, + }) + }), + ) +}) + +/** One page of the org's audit log, keyed by cursor + filters + the pinned ceiling. */ +export const auditLogPageAtom = (input: AuditLogPageInput) => + family(`${input.actorType ?? ""}|${input.outcome ?? ""}|${input.until ?? ""}|${input.cursor ?? ""}`) diff --git a/apps/web/src/routes/settings.tsx b/apps/web/src/routes/settings.tsx index 91ecda7a9..e2442ade8 100644 --- a/apps/web/src/routes/settings.tsx +++ b/apps/web/src/routes/settings.tsx @@ -8,6 +8,7 @@ import { BillingSection } from "@/components/settings/billing-section" import { MembersSection } from "@/components/settings/members-section" import { IngestionSection } from "@/components/settings/ingestion-section" import { ApiKeysSection } from "@/components/settings/api-keys-section" +import { AuditLogSection } from "@/components/settings/audit-log-section" import { DeveloperSection } from "@/components/settings/developer-section" import { McpSection } from "@/components/settings/mcp-section" import { NotificationsSection } from "@/components/settings/notifications-section" @@ -135,6 +136,7 @@ function SettingsPage() { {activeTab === "organization" && } {activeTab === "members" && } + {activeTab === "audit-log" && } {activeTab === "setup-audit" && } {activeTab === "ingestion" && } {activeTab === "api-keys" && } diff --git a/packages/domain/package.json b/packages/domain/package.json index a464a3756..ce89d63d7 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -3,6 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", + "sideEffects": false, "exports": { ".": "./src/index.ts", "./anticipated-errors": "./src/anticipated-errors.ts", diff --git a/packages/domain/src/clickhouse/migrations/0027_audit_log.ts b/packages/domain/src/clickhouse/migrations/0027_audit_log.ts new file mode 100644 index 000000000..f94c14e70 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0027_audit_log.ts @@ -0,0 +1,50 @@ +/** + * 0027 — `audit_log`: the org-wide audit trail, moved out of Postgres. + * + * Written only by the API worker through the managed Tinybird pipeline and read + * only by the admin-gated `GET /v2/audit_log`. It ships in the migration set so + * every ClickHouse the schema is applied to mirrors the managed table; a + * BYO-ClickHouse org never reads or writes it — reads are pinned to the managed + * route (`INGEST_PINNED_TABLES`). + * + * `requiredForIngest: false`: the ingest gateway writes nothing here, so the + * table's presence must not gate an org's ingest readiness. + * + * Retention is six years (HIPAA §164.316(b)(2)); `''` stands in for absent + * values throughout — see the datasource definition for the column contract. + */ +export const migration_0027_audit_log = { + version: 27, + description: "Create audit_log, the org-wide audit trail (actions, denials, and telemetry reads).", + requiredForIngest: false, + statements: [ + `CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index dfabc0c23..311ac48f2 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -32,6 +32,7 @@ import { migration_0023_service_operations_discriminators } from "./0023_service import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_index_filter_columns" +import { migration_0027_audit_log } from "./0027_audit_log" import { migration_0021_product_events } from "./0021_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" @@ -49,9 +50,10 @@ describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, ]) - expect(migrations.at(-1)).toBe(migration_0026_ai_trace_index_filter_columns) - expect(latestMigrationVersion).toBe(26) + expect(migrations.at(-1)).toBe(migration_0027_audit_log) + expect(latestMigrationVersion).toBe(27) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -60,7 +62,8 @@ describe("ClickHouse migrations", () => { // 0023 is the same: it only adds counter columns to those MV-populated // service-operations rollups. 0024 is read-path only too: `ai_trace_index` // is MV-populated and the gateway never writes it, and 0025 only rebuilds - // the three MV-populated service-overview views. + // the three MV-populated service-overview views. 0027 (`audit_log`) is + // written by the API worker through Tinybird, never by the gateway. expect(clickHouseSchemaVersion).toBe("21") expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) @@ -76,6 +79,7 @@ describe("ClickHouse migrations", () => { expect(migration_0025_commit_sha_vcs_revision.requiredForIngest).toBe(false) // 0026 widens the same MV-populated ai_trace_index and rebuilds its view. expect(migration_0026_ai_trace_index_filter_columns.requiredForIngest).toBe(false) + expect(migration_0027_audit_log.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 2bae7577c..bbf6eb295 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -25,6 +25,7 @@ import { migration_0023_service_operations_discriminators } from "./0023_service import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_index_filter_columns" +import { migration_0027_audit_log } from "./0027_audit_log" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -82,6 +83,7 @@ export const migrations: ReadonlyArray = [ migration_0024_ai_trace_index, migration_0025_commit_sha_vcs_revision, migration_0026_ai_trace_index_filter_columns, + migration_0027_audit_log, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index ee0379cba..05306fac3 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,13 +1,14 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "fab3e18c388b21aa3ed50bdda5bcd00f0458a4e52a9c7b0cf00dfd0ac3f0b17b" as const +export const projectRevision = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_trace_index (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SessionId String,\n VendorId LowCardinality(String),\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Model LowCardinality(String),\n AgentName LowCardinality(String),\n ToolName LowCardinality(String),\n SpanId String,\n ParentSpanId String,\n Duration UInt64,\n IsError UInt8,\n IsLlmCall UInt8,\n IsToolCall UInt8,\n Tokens Float64,\n Cost Float64\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS attribute_keys_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, Hour, AttributeKey)\nTTL Hour + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS attribute_values_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue)\nTTL Hour + INTERVAL 90 DAY", + "CREATE TABLE IF NOT EXISTS audit_log (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n RecordedAt DateTime64(3),\n ActorType LowCardinality(String),\n UserId String,\n ApiKeyId String,\n ActorId String,\n ActorLabel String,\n AffectedUserId String,\n Source LowCardinality(String),\n Action LowCardinality(String),\n Outcome LowCardinality(String),\n DenialReason String,\n ResourceType LowCardinality(String),\n ResourceId String,\n ChangedFields Array(String),\n Changes String,\n Metadata String,\n RequestId String,\n OriginIp String,\n OriginCountry LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 2190 DAY", "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime),\n ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL Minute + INTERVAL 90 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 1231042d6..1899d65ea 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "fab3e18c388b21aa3ed50bdda5bcd00f0458a4e52a9c7b0cf00dfd0ac3f0b17b" as const +export const projectRevision = "9fcd645645edaba7831f8417ebeb41b8d5b888fe1f820ea21d237488676d4ced" as const export const datasources = [ { @@ -24,6 +24,11 @@ export const datasources = [ content: 'DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, AttributeKey, Hour, AttributeValue"\nENGINE_TTL "Hour + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', }, + { + name: "audit_log", + content: + 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields[:]`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', + }, { name: "error_events", content: diff --git a/packages/domain/src/http/audit-log.ts b/packages/domain/src/http/audit-log.ts new file mode 100644 index 000000000..6ed98fada --- /dev/null +++ b/packages/domain/src/http/audit-log.ts @@ -0,0 +1,80 @@ +import { Context, Schema } from "effect" +import { HttpTaggedError } from "./error-policy" + +/** + * Who performed an audited action. `user` is a dashboard session, `api_key` a + * v1/v2 public-API credential, `agent` a registered LLM agent acting over MCP, + * and `system` Maple itself (crons, sweeps, lifecycle automation). + */ +export const AuditActorType = Schema.Literals(["user", "api_key", "agent", "system"]).annotate({ + identifier: "@maple/AuditActorType", + title: "Audit Actor Type", +}) +export type AuditActorType = Schema.Schema.Type + +/** Which surface the audited request arrived through. */ +export const AuditLogSource = Schema.Literals(["dashboard", "api", "mcp", "system"]).annotate({ + identifier: "@maple/AuditLogSource", + title: "Audit Log Source", +}) +export type AuditLogSource = Schema.Schema.Type + +/** Whether the action was performed or refused — denied attempts are logged too. */ +export const AuditOutcome = Schema.Literals(["allowed", "denied"]).annotate({ + identifier: "@maple/AuditOutcome", + title: "Audit Outcome", +}) +export type AuditOutcome = Schema.Schema.Type + +/** Before/after diff of an update, with the touched field names queryable on their own. */ +export const AuditChanges = Schema.Struct({ + fields: Schema.Array(Schema.String), + before: Schema.Record(Schema.String, Schema.Unknown), + after: Schema.Record(Schema.String, Schema.Unknown), +}).annotate({ + identifier: "@maple/AuditChanges", + title: "Audit Changes", +}) +export type AuditChanges = Schema.Schema.Type + +/** + * The audit action a data-read endpoint records. Telemetry (traces, logs, + * metrics, error events) and session replays are the two surfaces that can + * carry customer end-user data, so every read of them is logged — HIPAA audit + * controls cover access, not only change. + */ +export const AuditReadAction = Schema.Literals(["telemetry.read", "session_replay.read"]).annotate({ + identifier: "@maple/AuditReadAction", + title: "Audit Read Action", +}) +export type AuditReadAction = Schema.Schema.Type + +/** + * Endpoint/group annotation declaring that a successful call is a data read + * worth an audit entry. The auth middlewares consult it on every request; an + * endpoint without it (configuration, billing, the audit log itself) records + * nothing on reads. Declared here, next to the contracts, so "which endpoints + * expose telemetry" is visible where the endpoints are. + */ +export class AuditedRead extends Context.Reference( + "@maple/http/AuditedRead", + { defaultValue: () => undefined }, +) {} + +export class AuditLogPersistenceError extends HttpTaggedError()( + "@maple/http/errors/AuditLogPersistenceError", + { + message: Schema.String, + // Diagnostic only — `exposure: "redacted"` keeps it off the wire. + cause: Schema.optionalKey(Schema.Defect()), + }, + { + status: 503, + code: "audit_log_unavailable", + title: "The audit log is temporarily unavailable", + message: "The audit log is temporarily unavailable. Retry in a few seconds.", + retry: "backoff", + recovery: "retry", + exposure: "redacted", + }, +) {} diff --git a/packages/domain/src/http/errors.ts b/packages/domain/src/http/errors.ts index 89db688fd..ff19c7d84 100644 --- a/packages/domain/src/http/errors.ts +++ b/packages/domain/src/http/errors.ts @@ -16,6 +16,7 @@ import { TraceId, UserId, } from "../primitives" +import { AuditedRead } from "./audit-log" import { Authorization } from "./current-tenant" import { AlertSeverity } from "./alerts" import { @@ -881,7 +882,7 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueListQuery, success: ErrorIssuesListResponse, error: ErrorPersistenceError, - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("getIssue", "/issues/:issueId", { @@ -889,7 +890,7 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueDetailQuery, success: ErrorIssueDetailResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.post("transitionIssue", "/issues/:issueId/transitions", { @@ -985,20 +986,20 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueEventsQuery, success: ErrorIssueEventsResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("listIssueIncidents", "/issues/:issueId/incidents", { params: { issueId: ErrorIssueId }, success: ErrorIncidentsListResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("listOpenIncidents", "/incidents", { success: ErrorIncidentsListResponse, error: ErrorPersistenceError, - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.post("registerAgent", "/agents", { diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 9bcacd800..5d9d6e87f 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -5,6 +5,7 @@ export * from "./ai-triage" export * from "./investigations" export * from "./anomalies" export * from "./api-keys" +export * from "./audit-log" export * from "./alerts" export * from "./mobile-devices" export * from "./auth" diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 08440fca5..e6368b324 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -20,6 +20,7 @@ import { QueryEngineExecuteResponse, TinybirdDateTime, } from "../query-engine" +import { AuditedRead } from "./audit-log" import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { warehouseHttpErrors } from "./warehouse" @@ -2722,4 +2723,6 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") }), ) .prefix("/internal/query-engine") - .middleware(SessionAuthorization) {} + .middleware(SessionAuthorization) + // Every endpoint here reads telemetry for the dashboard. + .annotate(AuditedRead, "telemetry.read") {} diff --git a/packages/domain/src/http/session-replay.ts b/packages/domain/src/http/session-replay.ts index b2583ef05..749c04219 100644 --- a/packages/domain/src/http/session-replay.ts +++ b/packages/domain/src/http/session-replay.ts @@ -2,6 +2,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" import { Schema } from "effect" import { SessionId, TraceId, UserId } from "../primitives" import { TinybirdDateTime } from "../query-engine" +import { AuditedRead } from "./audit-log" import { Authorization, SessionAuthorization } from "./current-tenant" import { QueryEngineExecutionError, QueryEngineTimeoutError } from "./query-engine" import { warehouseHttpErrors } from "./warehouse" @@ -330,7 +331,8 @@ export class SessionReplaysApiGroup extends HttpApiGroup.make("sessionReplays") }), ) .prefix("/api/session-replays") - .middleware(Authorization) {} + .middleware(Authorization) + .annotate(AuditedRead, "session_replay.read") {} /** * Session-replay helpers that exist for the dashboard and are not public API. @@ -356,4 +358,5 @@ export class SessionReplaysInternalApiGroup extends HttpApiGroup.make("sessionRe }), ) .prefix("/internal/session-replays") - .middleware(SessionAuthorization) {} + .middleware(SessionAuthorization) + .annotate(AuditedRead, "session_replay.read") {} diff --git a/packages/domain/src/http/v2/api.ts b/packages/domain/src/http/v2/api.ts index 53ec8e161..e0814e060 100644 --- a/packages/domain/src/http/v2/api.ts +++ b/packages/domain/src/http/v2/api.ts @@ -6,6 +6,7 @@ import { V2AlertIncidentsApiGroup } from "./alert-incidents" import { V2AlertRulesApiGroup } from "./alert-rules" import { V2ApiKeysApiGroup } from "./api-keys" import { V2AttributeMappingsApiGroup } from "./attribute-mappings" +import { V2AuditLogApiGroup } from "./audit-log" import { V2DashboardsApiGroup } from "./dashboards" import { V2IngestKeysApiGroup } from "./ingest-keys" import { V2SlackIntegrationsApiGroup } from "./integrations" @@ -94,6 +95,7 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2PlanetScaleIntegrationsApiGroup) .add(V2ErrorIssuesApiGroup) .add(V2AttributeMappingsApiGroup) + .add(V2AuditLogApiGroup) .add(V2ScrapeTargetsApiGroup) .add(V2InstrumentationRecommendationsApiGroup) .add(V2InstrumentationAuditApiGroup) diff --git a/packages/domain/src/http/v2/audit-log.ts b/packages/domain/src/http/v2/audit-log.ts new file mode 100644 index 000000000..059532c95 --- /dev/null +++ b/packages/domain/src/http/v2/audit-log.ts @@ -0,0 +1,252 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" +import { Schema } from "effect" +import { AuditLogEntryId } from "../../primitives" +import { + AuditActorType, + AuditLogPersistenceError, + AuditLogSource, + AuditOutcome, +} from "../audit-log" +import { AuthorizationV2 } from "./auth" +import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" +import { V2InsufficientPermissions, V2ParameterInvalid } from "./errors" +import { publicErrors } from "./public-error" +import { PublicId, PublicIdPrefixes } from "./public-id" + +/** `alog_…` public ID ⇄ internal `AuditLogEntryId` (raw UUID). */ +export const AuditLogEntryPublicId = PublicId(PublicIdPrefixes.auditLogEntry, AuditLogEntryId) + +const actorTypeField = AuditActorType.annotate({ + description: + "Who performed the action: `user` (a dashboard session), `api_key` (a public-API credential), `agent` (a registered LLM agent acting over MCP), or `system` (Maple automation).", + examples: ["user"], +}) + +const sourceField = AuditLogSource.annotate({ + description: + "The surface the request arrived through: `dashboard`, `api` (the public v1/v2 API), `mcp`, or `system`.", + examples: ["dashboard"], +}) + +const outcomeField = AuditOutcome.annotate({ + description: + "Whether the action was performed (`allowed`) or refused (`denied`). Denied attempts — e.g. an API key lacking the required scope — are logged too.", + examples: ["allowed"], +}) + +export const V2AuditChanges = Schema.Struct({ + fields: Schema.Array(Schema.String).annotate({ + description: "Names of the fields the update touched.", + examples: [["name"]], + }), + before: Schema.Record(Schema.String, Schema.Unknown).annotate({ + description: "Prior values of the touched fields.", + }), + after: Schema.Record(Schema.String, Schema.Unknown).annotate({ + description: "New values of the touched fields.", + }), +}).annotate({ + identifier: "AuditLogChanges", + title: "Audit Log Changes", + description: "The before/after diff an update applied, keyed by field name.", +}) +export type V2AuditChanges = Schema.Schema.Type + +const auditLogEntryExample = { + id: "alog_4CzLmR1pTxWvYbNhQd82Kf", + object: "audit_log_entry", + action: "alert_rule.updated", + outcome: "allowed", + denial_reason: null, + actor_type: "user", + actor_id: "user_2fj3K9dLqWm8xYbT", + actor_name: "David", + actor_avatar_url: "https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCJ9", + affected_user: null, + source: "dashboard", + resource_type: "alert_rule", + resource_id: "alrt_YofPTrK9782DWwcnXhpcCw", + changes: { fields: ["name"], before: { name: "Errors" }, after: { name: "High error rate" } }, + metadata: null, + request_id: "8f2c1a9d4b7e3f60", + origin_ip: "203.0.113.7", + origin_country: "DE", + occurred_at: "2026-08-29T09:12:00.000Z", + recorded_at: "2026-08-29T09:12:00.412Z", +} as const + +// v2 wire schemas are annotated `Schema.Struct`s (not `Schema.Class`) — see the +// note in api-keys.ts. +export const V2AuditLogEntry = Schema.Struct({ + id: AuditLogEntryPublicId, + object: Schema.Literal("audit_log_entry").annotate({ + description: 'The object type — always `"audit_log_entry"`.', + examples: ["audit_log_entry"], + }), + action: Schema.String.annotate({ + description: "What happened, as `.` (e.g. `alert_rule.created`, `api_key.rolled`).", + examples: ["alert_rule.created"], + }), + outcome: outcomeField, + denial_reason: Schema.NullOr(Schema.String).annotate({ + description: "Why the action was refused, when `outcome` is `denied`; otherwise `null`.", + }), + actor_type: actorTypeField, + actor_id: Schema.NullOr(Schema.String).annotate({ + description: + "Public identifier of the actor: a `user_…` ID for users, a `key_…` ID for API keys, an `actor_…` ID for agents, or `null` for system actions.", + examples: ["user_2fj3K9dLqWm8xYbT"], + }), + actor_name: Schema.NullOr(Schema.String).annotate({ + description: + "Display name of the actor at the time of the action (agent name, API key name, …), or `null` when none was recorded.", + examples: ["David"], + }), + actor_avatar_url: Schema.NullOr(Schema.String).annotate({ + description: + "Avatar for a user actor, resolved from the workspace directory when the log is read. `null` for every non-user actor, and for a user who is no longer a member.", + examples: ["https://img.clerk.com/eyJ0eXBlIjoi…"], + }), + affected_user: Schema.NullOr(Schema.String).annotate({ + description: + "The `user_…` ID of the user the action was performed on (e.g. a removed member), when different from the actor; otherwise `null`.", + }), + source: sourceField, + resource_type: Schema.NullOr(Schema.String).annotate({ + description: "The kind of resource acted on (e.g. `alert_rule`, `dashboard`), or `null`.", + examples: ["alert_rule"], + }), + resource_id: Schema.NullOr(Schema.String).annotate({ + description: "Public ID of the resource acted on, or `null`.", + examples: ["alrt_YofPTrK9782DWwcnXhpcCw"], + }), + changes: Schema.NullOr(V2AuditChanges).annotate({ + description: "The before/after diff for updates, or `null` when the action carries no diff.", + }), + metadata: Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown)).annotate({ + description: "Action-specific context recorded with the entry, or `null`.", + }), + request_id: Schema.NullOr(Schema.String).annotate({ + description: + "Identifier of the HTTP request that performed the action, shared by every entry the request produced; or `null`.", + }), + origin_ip: Schema.NullOr(Schema.String).annotate({ + description: "Client IP the request originated from, or `null`.", + }), + origin_country: Schema.NullOr(Schema.String).annotate({ + description: "ISO 3166-1 country code the request originated from, or `null`.", + }), + occurred_at: Timestamp.annotate({ description: "When the action happened." }), + recorded_at: Timestamp.annotate({ + description: + "When the entry was durably recorded. Trails `occurred_at` by the audit pipeline's delivery latency.", + }), +}).annotate({ + identifier: "AuditLogEntry", + title: "Audit Log Entry", + description: + "One entry in the organization's append-only audit log: an allowed or denied action performed by a user, API key, or agent against a Maple resource.", + examples: [wireExample(auditLogEntryExample)], +}) +export type V2AuditLogEntry = Schema.Schema.Type + +/** Audit-log list query: standard pagination plus actor/action/resource/outcome/time filters. */ +export const V2AuditLogQuery = Schema.Struct({ + ...ListQuery.fields, + actor_type: Schema.optional( + AuditActorType.annotate({ + description: "Only return entries performed by this kind of actor.", + }), + ), + actor_id: Schema.optional( + Schema.String.annotate({ + description: + "Only return entries performed by this specific actor: a `user_…` user ID, `key_…` API key ID, or `actor_…` agent ID.", + }), + ), + affected_user: Schema.optional( + Schema.String.annotate({ + description: "Only return entries that acted on this `user_…` user.", + }), + ), + action: Schema.optional( + Schema.String.annotate({ + description: "Only return entries with exactly this action (e.g. `alert_rule.created`).", + }), + ), + outcome: Schema.optional( + AuditOutcome.annotate({ + description: "Only return entries with this outcome.", + }), + ), + resource_type: Schema.optional( + Schema.String.annotate({ + description: "Only return entries acting on this kind of resource (e.g. `dashboard`).", + }), + ), + resource_id: Schema.optional( + Schema.String.annotate({ + description: + "Only return entries acting on this exact resource, by its public ID (e.g. `dash_…`).", + }), + ), + changed: Schema.optional( + Schema.String.annotate({ + description: "Only return entries whose update touched this field name (e.g. `scopes`).", + }), + ), + request_id: Schema.optional( + Schema.String.annotate({ + description: "Only return entries produced by this HTTP request.", + }), + ), + since: Schema.optional( + Timestamp.annotate({ + description: "Only return entries that occurred at or after this time.", + }), + ), + until: Schema.optional( + Timestamp.annotate({ + description: "Only return entries that occurred at or before this time.", + }), + ), +}).annotate({ + identifier: "AuditLogQuery", + title: "Audit log query", + description: + "Pagination plus optional actor, action, outcome, resource, changed-field, request, and time-window filters.", +}) +export type V2AuditLogQuery = Schema.Schema.Type + +const [auditLogPersistence] = publicErrors(AuditLogPersistenceError) + +const AuditLogEntryList = ListOf(V2AuditLogEntry).annotate({ + identifier: "AuditLogEntryList", + title: "Audit log entry list", + description: "A cursor-paginated page of audit log entries, newest first.", +}) + +export class V2AuditLogApiGroup extends HttpApiGroup.make("auditLog") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2AuditLogQuery, + success: AuditLogEntryList, + error: [V2ParameterInvalid.schema, V2InsufficientPermissions.schema, auditLogPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listAuditLogEntries", + summary: "List audit log entries", + description: + "Returns your organization's audit log, newest first, optionally filtered by actor, action, outcome, resource, changed field, request, and time window. Cursor-paginated. Session callers must be organization administrators; API keys require the `audit_log:read` scope.", + }), + ), + ) + .prefix("/v2/audit_log") + .middleware(AuthorizationV2) + .annotateMerge( + OpenApi.annotations({ + title: "Audit Log", + description: + "The organization's append-only audit trail — allowed and denied actions performed through the dashboard, the public API, and MCP, attributed to the user, API key, or agent that performed them, with before/after diffs for updates. Reading it requires organization-administrator access (or the `audit_log:read` scope for API keys).", + }), + ) {} diff --git a/packages/domain/src/http/v2/error-issues.ts b/packages/domain/src/http/v2/error-issues.ts index f56696499..ec48fd0f1 100644 --- a/packages/domain/src/http/v2/error-issues.ts +++ b/packages/domain/src/http/v2/error-issues.ts @@ -10,6 +10,7 @@ import { WorkflowState, } from "../errors" import { SpanId, TraceId, UserId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { ListOf, ListQuery, Timestamp } from "./envelopes" import { V2CursorInvalid, V2CursorSortMismatch } from "./errors" @@ -242,6 +243,7 @@ export class V2ErrorIssuesApiGroup extends HttpApiGroup.make("errorIssues") ) .prefix("/v2/error_issues") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Error Issues", diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index e8b7d3ab1..b04facc3b 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -6,6 +6,7 @@ export * from "./anomalies" export * from "./api" export * from "./api-keys" export * from "./attribute-mappings" +export * from "./audit-log" export * from "./auth" export * from "./dashboards" export * from "./envelopes" diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index 9d0dfa744..e0ee0573c 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -118,6 +118,7 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/api_keys/{id}", "GET /v2/attribute_mappings", "GET /v2/attribute_mappings/{id}", + "GET /v2/audit_log", "GET /v2/dashboards", "GET /v2/dashboards/templates", "GET /v2/dashboards/{id}", diff --git a/packages/domain/src/http/v2/public-id.ts b/packages/domain/src/http/v2/public-id.ts index 7d232900c..b8f4c49b2 100644 --- a/packages/domain/src/http/v2/public-id.ts +++ b/packages/domain/src/http/v2/public-id.ts @@ -28,6 +28,7 @@ export const PublicIdPrefixes = { alertDestination: "dest", alertIncident: "inc", actor: "actor", + auditLogEntry: "alog", errorIssue: "iss", errorIncident: "einc", investigation: "inv", diff --git a/packages/domain/src/http/v2/session-replays.ts b/packages/domain/src/http/v2/session-replays.ts index cad31dfeb..1ad0f1354 100644 --- a/packages/domain/src/http/v2/session-replays.ts +++ b/packages/domain/src/http/v2/session-replays.ts @@ -1,6 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" import { Schema } from "effect" import { SessionId, TraceId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" import { defineV2Error, V2ParameterInvalid } from "./errors" @@ -578,6 +579,7 @@ export class V2SessionReplaysApiGroup extends HttpApiGroup.make("sessionReplays" ) .prefix("/v2/session_replays") .middleware(AuthorizationV2) + .annotate(AuditedRead, "session_replay.read") .annotateMerge( OpenApi.annotations({ title: "Session Replays", diff --git a/packages/domain/src/http/v2/telemetry.ts b/packages/domain/src/http/v2/telemetry.ts index 80b37e0b4..6732ade05 100644 --- a/packages/domain/src/http/v2/telemetry.ts +++ b/packages/domain/src/http/v2/telemetry.ts @@ -1,6 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" import { Schema } from "effect" import { MetricName, ServiceName, SpanId, TraceId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" import { defineV2Error, V2CursorInvalid, V2ParameterInvalid, V2TimeRangeInvalid } from "./errors" @@ -765,6 +766,7 @@ export class V2TracesApiGroup extends HttpApiGroup.make("traces") ) .prefix("/v2/traces") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Traces", @@ -867,6 +869,7 @@ export class V2LogsApiGroup extends HttpApiGroup.make("logs") ) .prefix("/v2/logs") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Logs", @@ -947,6 +950,7 @@ export class V2MetricsApiGroup extends HttpApiGroup.make("metrics") ) .prefix("/v2/metrics") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Metrics", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 765fcd259..825ac1af5 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1583,6 +1583,64 @@ export const alertChecks = defineDatasource("alert_checks", { export type AlertChecksRow = InferRow +/** + * The org-wide audit log: one row per allowed or denied action, and per read + * of telemetry or session replays, attributed to the user, API key, agent, or + * Maple itself that performed it. Written by the API through `ingest` (the + * audit events queue consumer, or the producer directly when no queue is + * bound) and read only by the admin-gated `GET /v2/audit_log`; never fed by a + * materialized view and never routed to a BYO ClickHouse — the log is Maple's + * record, not the customer warehouse's. + * + * Absent values are empty strings rather than NULL: `LowCardinality(Nullable)` + * is awkward in ClickHouse and every read maps `''` back to `null` on the wire. + * `Changes`/`Metadata` hold JSON documents (`''` when none); `ChangedFields` + * keeps the touched field names queryable without parsing `Changes`. + * + * Six-year retention: HIPAA §164.316(b)(2) keeps required documentation for six + * years, and the audit trail is the documentation of who accessed what. + */ +export const auditLog = defineDatasource("audit_log", { + description: + "Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.", + schema: { + OrgId: t.string().lowCardinality(), + Id: t.string(), + OccurredAt: t.dateTime64(3), + RecordedAt: t.dateTime64(3), + ActorType: t.string().lowCardinality(), + UserId: t.string(), + ApiKeyId: t.string(), + ActorId: t.string(), + ActorLabel: t.string(), + AffectedUserId: t.string(), + Source: t.string().lowCardinality(), + Action: t.string().lowCardinality(), + Outcome: t.string().lowCardinality(), + DenialReason: t.string(), + ResourceType: t.string().lowCardinality(), + ResourceId: t.string(), + // `[:]` is what lets the Events API map a JSON array onto Array(String). + ChangedFields: column(t.array(t.string()), { jsonPath: "$.ChangedFields[:]" }), + Changes: t.string(), + Metadata: t.string(), + RequestId: t.string(), + OriginIp: t.string(), + OriginCountry: t.string().lowCardinality(), + }, + // ReplacingMergeTree keyed on the entry id makes queue redelivery idempotent: + // a second delivery of the same event collapses at the next merge. Until + // then a page can carry both copies; `AuditLogService.list` drops the + // repeat by id. + engine: engine.replacingMergeTree({ + partitionKey: "toYYYYMM(OccurredAt)", + sortingKey: ["OrgId", "OccurredAt", "Id"], + ttl: "toDate(OccurredAt) + INTERVAL 2190 DAY", + }), +}) + +export type AuditLogRow = InferRow + /** * Minute-grain operation metrics used by the service-detail Operations panel. * The operation name is normalized once by the write-side MV, while exact and diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index 64b0064bc..5e82a3e14 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -7,6 +7,8 @@ const RETENTION_DAYS = { // it past the source's own retention would store rows detection can no // longer cross-check against a raw trace. ai_trace_index: 30, + // Six years — HIPAA's documentation retention floor. Never rebuildable. + audit_log: 2190, attribute_keys_hourly: 90, attribute_values_hourly: 90, error_events: 90, diff --git a/packages/infra/src/cloudflare/worker-runtime.ts b/packages/infra/src/cloudflare/worker-runtime.ts index 65799bd06..b2bcbdb88 100644 --- a/packages/infra/src/cloudflare/worker-runtime.ts +++ b/packages/infra/src/cloudflare/worker-runtime.ts @@ -104,6 +104,12 @@ export const withRequestRuntime = , Ctx e * draining the scheduler first and registering the whole thing with * `ctx.waitUntil`. Rethrows so the CF runtime reports the failure. * + * `onSettled` runs once the runtime is disposed, inside the same `waitUntil` + * registration — for the telemetry flush every handler owes at the end of an + * invocation. Registering it here rather than in a caller's `finally` keeps it + * inside a `waitUntil` the platform has already accepted, and keeps the flush + * after dispose, where the last spans have been emitted. + * * `onInterrupt` decides what an interrupt-only exit (isolate teardown mid-run) * looks like to the caller: * - `"reject"` (default): rethrow, so the CF runtime reports the invocation as @@ -117,7 +123,10 @@ export const runScheduledEffect = ( layer: Layer.Layer, program: Effect.Effect, ctx: ExecutionContextLike, - options?: { readonly onInterrupt?: "reject" | "graceful" }, + options?: { + readonly onInterrupt?: "reject" | "graceful" + readonly onSettled?: () => Promise + }, ): Promise => { const runtime = ManagedRuntime.make(layer) const done = runtime @@ -135,6 +144,9 @@ export const runScheduledEffect = ( await runtime.dispose().catch((err) => { console.error("[worker-runtime] scheduled runtime dispose failed:", err) }) + await options?.onSettled?.().catch((err) => { + console.error("[worker-runtime] scheduled onSettled failed:", err) + }) }) ctx.waitUntil(done.catch(() => undefined)) return done diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index 4b8467d80..3c909ff7e 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -128,6 +128,9 @@ export type ActorId = Schema.Schema.Type export const ErrorIssueEventId = MapleUuidId("@maple/ErrorIssueEventId", "Error Issue Event ID") export type ErrorIssueEventId = Schema.Schema.Type +export const AuditLogEntryId = MapleUuidId("@maple/AuditLogEntryId", "Audit Log Entry ID") +export type AuditLogEntryId = Schema.Schema.Type + export const ErrorIssuePullRequestId = MapleUuidId( "@maple/ErrorIssuePullRequestId", "Error Issue Pull Request ID", diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 1188ea17b..9f925c0d7 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -22,6 +22,79 @@ SELECT GROUP BY orgId FORMAT JSON +-- builder:audit-log:auditLogEntriesQuery:default [906b6bee] +SELECT + Id AS id, + OccurredAt AS occurredAt, + RecordedAt AS recordedAt, + ActorType AS actorType, + UserId AS userId, + ApiKeyId AS apiKeyId, + ActorId AS actorId, + ActorLabel AS actorLabel, + AffectedUserId AS affectedUserId, + Source AS source, + Action AS action, + Outcome AS outcome, + DenialReason AS denialReason, + ResourceType AS resourceType, + ResourceId AS resourceId, + ChangedFields AS changedFields, + Changes AS changes, + Metadata AS metadata, + RequestId AS requestId, + OriginIp AS originIp, + OriginCountry AS originCountry + FROM audit_log + WHERE OrgId = 'org_sql_catalog' + ORDER BY occurredAt DESC, id DESC + LIMIT 50 + OFFSET 0 + FORMAT JSON + +-- builder:audit-log:auditLogEntriesQuery:filtered [a6bc921e] +SELECT + Id AS id, + OccurredAt AS occurredAt, + RecordedAt AS recordedAt, + ActorType AS actorType, + UserId AS userId, + ApiKeyId AS apiKeyId, + ActorId AS actorId, + ActorLabel AS actorLabel, + AffectedUserId AS affectedUserId, + Source AS source, + Action AS action, + Outcome AS outcome, + DenialReason AS denialReason, + ResourceType AS resourceType, + ResourceId AS resourceId, + ChangedFields AS changedFields, + Changes AS changes, + Metadata AS metadata, + RequestId AS requestId, + OriginIp AS originIp, + OriginCountry AS originCountry + FROM audit_log + WHERE OrgId = 'org_sql_catalog' + AND ActorType = 'user' + AND UserId = 'user_1' + AND ApiKeyId = 'key_1' + AND ActorId = 'actor_1' + AND AffectedUserId = 'user_2' + AND Action = 'dashboard.updated' + AND Outcome = 'allowed' + AND ResourceType = 'dashboard' + AND ResourceId = 'dash_1' + AND has(ChangedFields, 'name') + AND RequestId = 'ray' + AND OccurredAt >= '2026-01-01 10:30:00' + AND OccurredAt <= '2026-01-03 14:15:00' + ORDER BY occurredAt DESC, id DESC + LIMIT 50 + OFFSET 50 + FORMAT JSON + -- builder:containers:containerCountersSummaryQuery:default [6bbc043d] SELECT avg(memoryBytesAvg) AS memoryBytesAvg, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 4181b25b0..587e00c20 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -315,6 +315,56 @@ const productEventsFixtures: ReadonlyArray = [ export const builderFixtures: ReadonlyArray = [ ...productEventsFixtures, + // Audit log listing (apps/api/src/services/audit/AuditLogService.ts `list`). + { + module: "audit-log", + name: "auditLogEntriesQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.auditLogEntriesQuery({ limit: 50, offset: 0 }), { orgId: ORG_ID }), + }, + { + // Every optional filter bound at once, including the raw `has(...)` clause. + module: "audit-log", + name: "auditLogEntriesQuery", + label: "filtered", + compile: () => + CH.compileUnsafe( + CH.auditLogEntriesQuery({ + actorType: true, + userId: true, + apiKeyId: true, + actorId: true, + affectedUserId: true, + action: true, + outcome: true, + resourceType: true, + resourceId: true, + changedField: true, + requestId: true, + since: true, + until: true, + limit: 50, + offset: 50, + }), + { + orgId: ORG_ID, + actorType: "user", + userId: "user_1", + apiKeyId: "key_1", + actorId: "actor_1", + affectedUserId: "user_2", + action: "dashboard.updated", + outcome: "allowed", + resourceType: "dashboard", + resourceId: "dash_1", + changedField: "name", + requestId: "ray", + since: START_TIME, + until: END_TIME, + }, + ), + }, // Session replay fixtures used by the replay routes. { module: "session-replays", diff --git a/packages/query-engine/src/benchmark/catalog.test.ts b/packages/query-engine/src/benchmark/catalog.test.ts index 41c8d7856..3f54a9ab6 100644 --- a/packages/query-engine/src/benchmark/catalog.test.ts +++ b/packages/query-engine/src/benchmark/catalog.test.ts @@ -23,6 +23,7 @@ import { import { builderFixtures } from "./builders" import * as activityQueries from "../ch/queries/activity" import * as alertCheckQueries from "../ch/queries/alert-checks" +import * as auditLogQueries from "../ch/queries/audit-log" import * as anomalyQueries from "../ch/queries/anomaly" import * as attributeKeyQueries from "../ch/queries/attribute-keys" import * as containerQueries from "../ch/queries/containers" @@ -253,6 +254,7 @@ describe("sql catalog", () => { const QUERY_MODULES: Record> = { activity: activityQueries, "alert-checks": alertCheckQueries, + "audit-log": auditLogQueries, anomaly: anomalyQueries, "attribute-keys": attributeKeyQueries, containers: containerQueries, diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 04b1e4433..e63cddba8 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -411,6 +411,13 @@ export { type AlertChecksSummaryOutput, } from "./queries/alert-checks" +// Queries — Audit log (org-wide audit trail, admin-only) +export { + auditLogEntriesQuery, + type AuditLogEntriesOpts, + type AuditLogEntriesOutput, +} from "./queries/audit-log" + // Queries — Cloudflare integration usage (integrations-page ingest proof) // Queries — Cloudflare service-map stats (per-zone / per-Worker node rollups) diff --git a/packages/query-engine/src/ch/queries/audit-log.ts b/packages/query-engine/src/ch/queries/audit-log.ts new file mode 100644 index 000000000..307c8885b --- /dev/null +++ b/packages/query-engine/src/ch/queries/audit-log.ts @@ -0,0 +1,108 @@ +import * as CH from "@maple-dev/clickhouse-builder/expr" +import { from, param, paramPlaceholder } from "@maple-dev/clickhouse-builder" +import { AuditLog } from "../tables" + +/** + * Which optional filters a listing applies. Every set flag binds a parameter of + * the same name at compile time; the values themselves never enter the SQL. + */ +export interface AuditLogEntriesOpts { + readonly actorType?: boolean + readonly userId?: boolean + readonly apiKeyId?: boolean + readonly actorId?: boolean + readonly affectedUserId?: boolean + readonly action?: boolean + readonly outcome?: boolean + readonly resourceType?: boolean + readonly resourceId?: boolean + readonly changedField?: boolean + readonly requestId?: boolean + readonly since?: boolean + readonly until?: boolean + readonly limit: number + readonly offset: number +} + +/** + * One org's audit log, newest first, offset-paginated. Pinned to the managed + * route: the table is written through `ingest` and does not exist in a BYO + * ClickHouse. A redelivered entry ReplacingMergeTree has not merged yet can + * appear twice here; the service collapses it by id. + */ +export function auditLogEntriesQuery(opts: AuditLogEntriesOpts) { + return from(AuditLog) + .select(($) => ({ + id: $.Id, + occurredAt: $.OccurredAt, + recordedAt: $.RecordedAt, + actorType: $.ActorType, + userId: $.UserId, + apiKeyId: $.ApiKeyId, + actorId: $.ActorId, + actorLabel: $.ActorLabel, + affectedUserId: $.AffectedUserId, + source: $.Source, + action: $.Action, + outcome: $.Outcome, + denialReason: $.DenialReason, + resourceType: $.ResourceType, + resourceId: $.ResourceId, + changedFields: $.ChangedFields, + changes: $.Changes, + metadata: $.Metadata, + requestId: $.RequestId, + originIp: $.OriginIp, + originCountry: $.OriginCountry, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + opts.actorType ? $.ActorType.eq(param.string("actorType")) : undefined, + opts.userId ? $.UserId.eq(param.string("userId")) : undefined, + opts.apiKeyId ? $.ApiKeyId.eq(param.string("apiKeyId")) : undefined, + opts.actorId ? $.ActorId.eq(param.string("actorId")) : undefined, + opts.affectedUserId ? $.AffectedUserId.eq(param.string("affectedUserId")) : undefined, + opts.action ? $.Action.eq(param.string("action")) : undefined, + opts.outcome ? $.Outcome.eq(param.string("outcome")) : undefined, + opts.resourceType ? $.ResourceType.eq(param.string("resourceType")) : undefined, + opts.resourceId ? $.ResourceId.eq(param.string("resourceId")) : undefined, + // Array membership has no builder verb yet; the placeholder keeps the + // value parameterised exactly like the typed comparisons above. + opts.changedField + ? CH.rawCond(`has(ChangedFields, ${paramPlaceholder("string", "changedField")})`) + : undefined, + opts.requestId ? $.RequestId.eq(param.string("requestId")) : undefined, + opts.since ? $.OccurredAt.gte(param.dateTimeString("since")) : undefined, + opts.until ? $.OccurredAt.lte(param.dateTimeString("until")) : undefined, + ]) + .orderBy(["occurredAt", "desc"], ["id", "desc"]) + .limit(opts.limit) + .offset(opts.offset) + .format("JSON") + .route("ingest") +} + +/** One listed entry as the warehouse returns it: `''` for absent values, JSON text for documents. */ +export interface AuditLogEntriesOutput { + readonly id: string + readonly occurredAt: string + readonly recordedAt: string + readonly actorType: string + readonly userId: string + readonly apiKeyId: string + readonly actorId: string + readonly actorLabel: string + readonly affectedUserId: string + readonly source: string + readonly action: string + readonly outcome: string + readonly denialReason: string + readonly resourceType: string + readonly resourceId: string + readonly changedFields: ReadonlyArray + readonly changes: string + readonly metadata: string + readonly requestId: string + readonly originIp: string + readonly originCountry: string +} diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 06c0c3088..93a59cf4d 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -620,6 +620,31 @@ export const AlertChecks = table("alert_checks", { ErrorCategory: T.string, }) +export const AuditLog = table("audit_log", { + OrgId: orgId, + Id: T.string, + OccurredAt: dateTime64, + RecordedAt: dateTime64, + ActorType: T.string, + UserId: T.string, + ApiKeyId: T.string, + ActorId: T.string, + ActorLabel: T.string, + AffectedUserId: T.string, + Source: T.string, + Action: T.string, + Outcome: T.string, + DenialReason: T.string, + ResourceType: T.string, + ResourceId: T.string, + ChangedFields: T.array(T.string), + Changes: T.string, + Metadata: T.string, + RequestId: T.string, + OriginIp: T.string, + OriginCountry: T.string, +}) + export const SessionReplays = table("session_replays", { OrgId: orgId, SessionId: T.string, diff --git a/packages/query-engine/src/datetime.ts b/packages/query-engine/src/datetime.ts index 25c4eef82..9d0f56195 100644 --- a/packages/query-engine/src/datetime.ts +++ b/packages/query-engine/src/datetime.ts @@ -158,6 +158,34 @@ export const WarehouseDateTime = Schema.String.pipe( }) export type WarehouseDateTime = Schema.Schema.Type +/** + * A warehouse `DateTime64(3)` literal: `YYYY-MM-DD HH:mm:ss.SSS`, UTC, no zone. + * + * The millisecond sibling of {@link WarehouseDateTime}, and branded for the same + * reason: the shapes a hand-built string reaches for — whole seconds, or ISO + * with `T`/`Z` — are both wrong here. Seconds collapse the sub-second ordering + * that a `DateTime64(3)` sort key exists to keep, and the Events API's JSONPath + * parser rejects `T`/`Z` outright, so an ingest row carrying one is dropped by + * the warehouse rather than by any check in front of it. + */ +export const WarehouseDateTime64 = Schema.String.pipe( + Schema.check(Schema.isPattern(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/)), + Schema.brand("@maple/WarehouseDateTime64"), +).annotate({ + title: "WarehouseDateTime64", + description: + "UTC warehouse DateTime64(3) literal, `YYYY-MM-DD HH:mm:ss.SSS` (e.g. `2026-08-25 08:47:52.041`).", +}) +export type WarehouseDateTime64 = Schema.Schema.Type + +/** + * Format epoch milliseconds as a {@link WarehouseDateTime64} — the one + * sanctioned way to mint the brand, mirroring {@link warehouseDateTime}. + */ +export function warehouseDateTime64(epochMs: number): WarehouseDateTime64 { + return WarehouseDateTime64.make(formatWarehouseDateTimeMs(epochMs)) +} + /** * Decodes any accepted timestamp input — ISO-8601 with `Z` or an offset, the * warehouse shape with or without fractional seconds, a bare date — into a diff --git a/packages/query-engine/src/execution/datasource-routing.ts b/packages/query-engine/src/execution/datasource-routing.ts index a23cfc091..9a42bca5d 100644 --- a/packages/query-engine/src/execution/datasource-routing.ts +++ b/packages/query-engine/src/execution/datasource-routing.ts @@ -8,7 +8,7 @@ * org-BYO backend while referencing one of these tables would silently return * empty rows, so it logs a warning instead of failing quietly. */ -export const INGEST_PINNED_TABLES: ReadonlyArray = ["alert_checks"] +export const INGEST_PINNED_TABLES: ReadonlyArray = ["alert_checks", "audit_log"] export const findIngestPinnedTable = (sql: string): string | undefined => INGEST_PINNED_TABLES.find((table) => sql.includes(table))