Skip to content

Commit f4780d7

Browse files
ericallamTrigger.dev RepoOps
ericallam
authored and
Trigger.dev RepoOps
committed
feat(webapp,webhook-engine,core): per-org webhook delivery retention
Webhook deliveries are kept per organization for 3, 7, 30, 90, 180 or 365 days. Deliveries are stored for at least 30 days, so upgrading a plan shows older history right away, and organizations can choose to have deliveries deleted at their retention. ```ts const { id, retentionDays } = WebhookDeliveryId.generate({ retentionDays: 30 }); WebhookDeliveryId.parseRetentionDays(id); // 30 ``` Mono-RevId: dd839153fe9e98e1bfdb258e863fb465a023a85a
1 parent 1f65ca0 commit f4780d7

35 files changed

Lines changed: 1277 additions & 352 deletions

File tree

‎apps/webapp/app/components/admin/backOffice/WebhookLimitsSection.server.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,23 @@ const OptionalLimit = z.preprocess(
99
z.coerce.number().int().min(1).max(2_147_483_647).optional()
1010
);
1111

12+
const OptionalRetention = z.preprocess(
13+
(value) => (value === "" || value === null ? undefined : value),
14+
z.coerce.number().int().min(1).max(365).optional()
15+
);
16+
17+
const OptionalFlag = z.preprocess(
18+
(value) => (value === "on" ? true : undefined),
19+
z.literal(true).optional()
20+
);
21+
1222
const SetWebhookLimitsSchema = z.object({
1323
intent: z.literal(WEBHOOK_LIMITS_INTENT),
1424
maxWaitersPerEnvironment: OptionalLimit,
1525
maxWaitersPerEndpoint: OptionalLimit,
1626
concurrency: OptionalLimit,
27+
deliveryRetentionDays: OptionalRetention,
28+
deliveryRetentionStrict: OptionalFlag,
1729
});
1830

1931
export type WebhookLimitsActionResult =

‎apps/webapp/app/components/admin/backOffice/WebhookLimitsSection.tsx‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Form } from "@remix-run/react";
22
import { useEffect, useState } from "react";
33
import { Button } from "~/components/primitives/Buttons";
4+
import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
45
import { FormError } from "~/components/primitives/FormError";
56
import { Header2 } from "~/components/primitives/Headers";
67
import { Hint } from "~/components/primitives/Hint";
@@ -15,7 +16,13 @@ export const WEBHOOK_LIMITS_SAVED_VALUE = "webhook-limits";
1516

1617
type FieldErrors = Record<string, string[] | undefined> | null;
1718

18-
const FIELDS: Array<{ key: keyof WebhookLimits; label: string; hint: string }> = [
19+
type NumericLimit =
20+
| "maxWaitersPerEnvironment"
21+
| "maxWaitersPerEndpoint"
22+
| "concurrency"
23+
| "deliveryRetentionDays";
24+
25+
const FIELDS: Array<{ key: NumericLimit; label: string; hint: string }> = [
1926
{
2027
key: "maxWaitersPerEnvironment",
2128
label: "Waiters per environment",
@@ -31,6 +38,11 @@ const FIELDS: Array<{ key: keyof WebhookLimits; label: string; hint: string }> =
3138
label: "Processing concurrency",
3239
hint: "Webhook jobs one environment can have in flight at once.",
3340
},
41+
{
42+
key: "deliveryRetentionDays",
43+
label: "Delivery retention (days)",
44+
hint: "Days of deliveries the org can see, 1 to 365. Older ones are hidden right away.",
45+
},
3446
];
3547

3648
type Props = {
@@ -103,6 +115,17 @@ export function WebhookLimitsSection({
103115
</Property.Value>
104116
</Property.Item>
105117
))}
118+
<Property.Item>
119+
<Property.Label>Delivery storage</Property.Label>
120+
<Property.Value>
121+
{limits.deliveryStorageDays.toLocaleString()} days
122+
<span className="text-text-dimmed">
123+
{limits.deliveryRetentionStrict
124+
? " (deleted at retention)"
125+
: " (hidden after retention)"}
126+
</span>
127+
</Property.Value>
128+
</Property.Item>
106129
</Property.Table>
107130
) : (
108131
<Form method="post" className="flex flex-col gap-3 pt-2">
@@ -121,6 +144,20 @@ export function WebhookLimitsSection({
121144
<FormError>{fieldError(field.key)}</FormError>
122145
</div>
123146
))}
147+
<div className="flex flex-col gap-1">
148+
<CheckboxWithLabel
149+
name="deliveryRetentionStrict"
150+
value="on"
151+
variant="simple/small"
152+
label="Delete deliveries at the retention"
153+
defaultChecked={overrides.deliveryRetentionStrict === true}
154+
/>
155+
<Hint>
156+
Off: deliveries are stored for at least 30 days and hidden after the retention, so an
157+
upgrade shows older history. On: they are stored only for the retention (rounded up to
158+
3, 7, 30, 90, 180 or 365 days). Applies to new deliveries.
159+
</Hint>
160+
</div>
124161
<div className="flex items-center gap-2">
125162
<Button type="submit" variant="primary/medium" disabled={isSubmitting}>
126163
Save

‎apps/webapp/app/env.server.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import { z } from "zod";
22
import { MachinePresetName } from "@trigger.dev/core/v3";
3-
import { parseNaturalLanguageDurationInMs } from "@trigger.dev/core/v3/isomorphic";
3+
import {
4+
parseNaturalLanguageDurationInMs,
5+
webhookDeliveryRetentionClass,
6+
} from "@trigger.dev/core/v3/isomorphic";
47
import { BoolEnv } from "./utils/boolEnv";
58
import { isValidDatabaseUrl } from "./utils/db";
69
import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server";
@@ -2036,7 +2039,15 @@ const EnvironmentSchema = z
20362039
WEBHOOK_PARTITION_ENSURE_SCHEDULE: z.string().optional(),
20372040
WEBHOOK_PARTITION_ENSURE_JITTER_MS: z.coerce.number().int().optional(),
20382041
WEBHOOK_PARTITION_LOOKAHEAD_DAYS: z.coerce.number().int().default(10),
2039-
WEBHOOK_PARTITION_RETENTION_DAYS: z.coerce.number().int().default(60),
2042+
WEBHOOK_DELIVERY_RETENTION_DAYS: z.coerce.number().int().positive().default(30),
2043+
WEBHOOK_DELIVERY_STORAGE_DAYS: z.coerce
2044+
.number()
2045+
.int()
2046+
.default(30)
2047+
.refine(
2048+
(days) => webhookDeliveryRetentionClass(days) !== undefined,
2049+
"WEBHOOK_DELIVERY_STORAGE_DAYS must be a delivery retention class (3, 7, 30, 90, 180 or 365)"
2050+
),
20402051

20412052
// Ingest hot-path cache for the endpoint + resolved signing secret (keyed by opaqueId). 0 disables.
20422053
WEBHOOK_ENDPOINT_CACHE_TTL_MS: z.coerce.number().int().default(30_000),
@@ -2238,7 +2249,7 @@ const EnvironmentSchema = z
22382249
SESSION_REPLICATION_INSERT_BASE_DELAY_MS: z.coerce.number().int().default(100),
22392250
SESSION_REPLICATION_INSERT_MAX_DELAY_MS: z.coerce.number().int().default(2000),
22402251

2241-
// Webhook deliveries replication (Postgres → ClickHouse webhook_deliveries_v1).
2252+
// Webhook deliveries replication (Postgres → ClickHouse webhook_deliveries_v2).
22422253
// Shares Redis with the runs replicator for leader locking but has its own
22432254
// slot and publication so the two consume independently. The source table is
22442255
// a partitioned parent, so the publication is created with

‎apps/webapp/app/presenters/v3/LimitsPresenter.server.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ export type LimitsResult = {
7777
webhookWaitersPerEnvironment: QuotaInfo | null;
7878
webhookWaitersPerEndpoint: QuotaInfo | null;
7979
webhookConcurrency: QuotaInfo | null;
80+
webhookDeliveryRetentionDays: QuotaInfo | null;
8081
};
8182
features: {
8283
hasStagingEnvironment: FeatureInfo;
@@ -148,7 +149,10 @@ export class LimitsPresenter extends BasePresenter {
148149
? await webhookEngine.countLiveWaiters(environmentId).catch(() => 0)
149150
: 0;
150151
const webhookQuota = (
151-
key: keyof typeof webhookLimits.limits,
152+
key: Exclude<
153+
keyof typeof webhookLimits.limits,
154+
"deliveryRetentionStrict" | "deliveryStorageDays"
155+
>,
152156
name: string,
153157
description: string,
154158
currentUsage: number
@@ -412,6 +416,12 @@ export class LimitsPresenter extends BasePresenter {
412416
"Maximum webhook deliveries and waiter resumes being processed at once in this environment. More wait their turn.",
413417
0
414418
),
419+
webhookDeliveryRetentionDays: webhookQuota(
420+
"deliveryRetentionDays",
421+
"Webhook delivery retention",
422+
"Number of days of webhook deliveries you can see, including their bodies and headers.",
423+
0
424+
),
415425
},
416426
features: {
417427
hasStagingEnvironment: {

‎apps/webapp/app/presenters/v3/WebhookEndpointsListPresenter.server.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ export class WebhookEndpointsListPresenter {
263263
toUnixTimestamp64Milli(max(delivery_created_at)) AS last_delivery_ms
264264
FROM (
265265
SELECT webhook_endpoint_id, delivery_id, argMax(status, _version) AS current_status, max(created_at) AS delivery_created_at
266-
FROM trigger_dev.webhook_deliveries_v1
266+
FROM trigger_dev.webhook_deliveries_v2
267267
WHERE organization_id = {organizationId: String}
268268
AND project_id = {projectId: String}
269269
AND environment_id = {environmentId: String}

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,7 @@ function QuotasSection({
550550
if (quotas.webhookWaitersPerEnvironment) quotaRows.push(quotas.webhookWaitersPerEnvironment);
551551
if (quotas.webhookWaitersPerEndpoint) quotaRows.push(quotas.webhookWaitersPerEndpoint);
552552
if (quotas.webhookConcurrency) quotaRows.push(quotas.webhookConcurrency);
553+
if (quotas.webhookDeliveryRetentionDays) quotaRows.push(quotas.webhookDeliveryRetentionDays);
553554

554555
const showSelfServe = useShowSelfServe();
555556

@@ -602,7 +603,10 @@ function QuotaRow({
602603
}) {
603604
// For log retention and query period, we don't show current usage as it's a duration, not a count
604605
// For widgets per dashboard, the usage varies per dashboard so we don't show a single number
605-
const isDurationQuota = quota.name === "Log retention" || quota.name === "Query period";
606+
const isDurationQuota =
607+
quota.name === "Log retention" ||
608+
quota.name === "Query period" ||
609+
quota.name === "Webhook delivery retention";
606610
const isPerItemQuota =
607611
quota.name === "Charts per dashboard" ||
608612
quota.name === "Webhook waiters per endpoint" ||

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.webhooks.deliveries.$deliveryParam/route.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import { DeliveryHeadersTable } from "~/components/webhookDeliveries/v1/Delivery
3232
import { DeliveryTargetsTable } from "~/components/webhookDeliveries/v1/DeliveryTargetsTable";
3333
import { DeliveryTimeline } from "~/components/webhookDeliveries/v1/DeliveryTimeline";
3434
import { $replica } from "~/db.server";
35-
import { env } from "~/env.server";
35+
import { webhookLimitsForEnvironment } from "~/v3/webhookLimits.server";
3636
import { useEnvironment } from "~/hooks/useEnvironment";
3737
import { useInterval } from "~/hooks/useInterval";
3838
import { useOrganization } from "~/hooks/useOrganizations";
@@ -125,7 +125,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
125125
eventJson,
126126
eventBytes,
127127
eventTruncatedForDisplay,
128-
retentionDays: env.WEBHOOK_PARTITION_RETENTION_DAYS,
128+
retentionDays: (await webhookLimitsForEnvironment(environment.id)).deliveryRetentionDays,
129129
});
130130
};
131131

‎apps/webapp/app/routes/admin.api.v1.webhooks.partitions.bootstrap.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ export async function action({ request }: ActionFunctionArgs) {
1414
const result = await bootstrapPartitions(webhookPartitionPrisma, {
1515
now: new Date(),
1616
lookaheadDays: env.WEBHOOK_PARTITION_LOOKAHEAD_DAYS,
17-
retentionDays: env.WEBHOOK_PARTITION_RETENTION_DAYS,
1817
});
1918

2019
return json({ success: true, ...result });

‎apps/webapp/app/services/webhookDeliveriesReplicationService.server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ export class WebhookDeliveriesReplicationService {
212212
publicationName: options.publicationName,
213213
table: "WebhookDelivery",
214214
additionalTables: ["WebhookEndpoint"],
215+
readdMainTable: true,
215216
publishViaPartitionRoot: options.publishViaPartitionRoot,
216217
redisOptions: options.redisOptions,
217218
autoAcknowledge: false,
@@ -932,6 +933,7 @@ function toWebhookDeliveryInsertArray(
932933
delivery.runId ?? "",
933934
delivery.status,
934935
delivery.isTest ? 1 : 0,
936+
delivery.retentionDays,
935937
delivery.createdAt.getTime(),
936938
delivery.updatedAt.getTime(),
937939
version.toString(),

0 commit comments

Comments
 (0)