feat(webapp,billing): per-environment webhook ingress limit weighted by body size

All of an environment's webhook endpoints now share a rate limit,
counted in tokens: a request costs one token for every 64 KiB of body it
carries, so larger deliveries count for more. Requests over the limit
get a `429` with `Retry-After`. The limit comes from the organization's
plan (300, 1,000 and 3,000 tokens every 10 seconds on Free, Hobby and
Pro), is shown on the limits page, and self-hosted installs set it with
`WEBHOOK_INGRESS_ENV_RATE_LIMIT_TOKENS` and
`WEBHOOK_INGRESS_ENV_RATE_LIMIT_WINDOW`.

Mono-RevId: 19aa15366f76a34a5d68767aa511d47e897e7177

Author: ericallam

.server-changes/webhook-environment-rate-limit.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Each environment's webhook endpoints now share a rate limit that counts larger request bodies as more than one request, set by the organization's plan and shown on the limits page.

apps/webapp/app/components/admin/backOffice/WebhookLimitsForm.ts

@@ -0,0 +1,47 @@
+import { z } from "zod";
+
+const OptionalLimit = z.preprocess(
+  (value) => (value === "" || value === null ? undefined : value),
+  z.coerce.number().int().min(1).max(2_147_483_647).optional()
+);
+
+const OptionalRetention = z.preprocess(
+  (value) => (value === "" || value === null ? undefined : value),
+  z.coerce.number().int().min(1).max(365).optional()
+);
+
+const OptionalWindowSeconds = z.preprocess(
+  (value) => (value === "" || value === null ? undefined : value),
+  z.coerce.number().int().min(1).max(3600).optional()
+);
+
+const OptionalBytes = z.preprocess(
+  (value) => (value === "" || value === null ? undefined : value),
+  z.coerce.number().int().min(1).max(Number.MAX_SAFE_INTEGER).optional()
+);
+
+const OptionalFlag = z.preprocess(
+  (value) => (value === "on" ? true : undefined),
+  z.literal(true).optional()
+);
+
+export const WEBHOOK_LIMITS_INTENT = "set-webhook-limits";
+
+const SetWebhookLimitsSchema = z.object({
+  intent: z.literal(WEBHOOK_LIMITS_INTENT),
+  maxWaitersPerEnvironment: OptionalLimit,
+  maxWaitersPerEndpoint: OptionalLimit,
+  concurrency: OptionalLimit,
+  deliveryRetentionDays: OptionalRetention,
+  deliveryRetentionStrict: OptionalFlag,
+  endpointRateLimitTokens: OptionalLimit,
+  endpointRateLimitWindowSeconds: OptionalWindowSeconds,
+  environmentRateLimitTokens: OptionalLimit,
+  environmentRateLimitWindowSeconds: OptionalWindowSeconds,
+  queuedPayloadBytes: OptionalBytes,
+});
+
+/** The back-office webhook limits form, parsed, or the field errors to show. */
+export function parseWebhookLimitsForm(formData: FormData) {
+  return SetWebhookLimitsSchema.safeParse(Object.fromEntries(formData));
+}

apps/webapp/app/components/admin/backOffice/WebhookLimitsSection.server.ts

@@ -1,39 +1,7 @@
 import { Prisma } from "@trigger.dev/database";
-import { z } from "zod";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
-import { WEBHOOK_LIMITS_INTENT } from "./WebhookLimitsSection";
-
-const OptionalLimit = z.preprocess(
-  (value) => (value === "" || value === null ? undefined : value),
-  z.coerce.number().int().min(1).max(2_147_483_647).optional()
-);
-
-const OptionalRetention = z.preprocess(
-  (value) => (value === "" || value === null ? undefined : value),
-  z.coerce.number().int().min(1).max(365).optional()
-);
-
-const OptionalWindowSeconds = z.preprocess(
-  (value) => (value === "" || value === null ? undefined : value),
-  z.coerce.number().int().min(1).max(3600).optional()
-);
-
-const OptionalFlag = z.preprocess(
-  (value) => (value === "on" ? true : undefined),
-  z.literal(true).optional()
-);
-
-const SetWebhookLimitsSchema = z.object({
-  intent: z.literal(WEBHOOK_LIMITS_INTENT),
-  maxWaitersPerEnvironment: OptionalLimit,
-  maxWaitersPerEndpoint: OptionalLimit,
-  concurrency: OptionalLimit,
-  deliveryRetentionDays: OptionalRetention,
-  deliveryRetentionStrict: OptionalFlag,
-  endpointRateLimitTokens: OptionalLimit,
-  endpointRateLimitWindowSeconds: OptionalWindowSeconds,
-});
+import { parseWebhookLimitsForm } from "./WebhookLimitsForm";
 
 export type WebhookLimitsActionResult =
   | { ok: true }
@@ -44,7 +12,7 @@ export async function handleWebhookLimitsAction(
   orgId: string,
   adminUserId: string
 ): Promise<WebhookLimitsActionResult> {
-  const submission = SetWebhookLimitsSchema.safeParse(Object.fromEntries(formData));
+  const submission = parseWebhookLimitsForm(formData);
   if (!submission.success) {
     return { ok: false, errors: submission.error.flatten().fieldErrors };
   }

apps/webapp/app/components/admin/backOffice/WebhookLimitsSection.tsx

@@ -1,4 +1,5 @@
 import { Form } from "@remix-run/react";
+import { WEBHOOK_LIMITS_INTENT } from "./WebhookLimitsForm";
 import { useEffect, useState } from "react";
 import { Button } from "~/components/primitives/Buttons";
 import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
@@ -11,7 +12,6 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import * as Property from "~/components/primitives/PropertyTable";
 import type { WebhookLimits, WebhookLimitsConfig } from "~/v3/webhookLimits";
 
-export const WEBHOOK_LIMITS_INTENT = "set-webhook-limits";
 export const WEBHOOK_LIMITS_SAVED_VALUE = "webhook-limits";
 
 type FieldErrors = Record<string, string[] | undefined> | null;
@@ -22,7 +22,10 @@ type NumericLimit =
   | "concurrency"
   | "deliveryRetentionDays"
   | "endpointRateLimitTokens"
-  | "endpointRateLimitWindowSeconds";
+  | "endpointRateLimitWindowSeconds"
+  | "environmentRateLimitTokens"
+  | "environmentRateLimitWindowSeconds"
+  | "queuedPayloadBytes";
 
 const FIELDS: Array<{ key: NumericLimit; label: string; hint: string }> = [
   {
@@ -55,6 +58,21 @@ const FIELDS: Array<{ key: NumericLimit; label: string; hint: string }> = [
     label: "Endpoint rate limit window (seconds)",
     hint: "The window the endpoint rate limit counts over, 1 to 3600 seconds.",
   },
+  {
+    key: "environmentRateLimitTokens",
+    label: "Environment rate limit (tokens)",
+    hint: "Tokens all of an environment's endpoints accept per window. A request costs one per started 64 KiB of body. More get 429 with Retry-After.",
+  },
+  {
+    key: "environmentRateLimitWindowSeconds",
+    label: "Environment rate limit window (seconds)",
+    hint: "The window the environment rate limit counts over, 1 to 3600 seconds.",
+  },
+  {
+    key: "queuedPayloadBytes",
+    label: "Queued payload limit (bytes)",
+    hint: "Payload bytes one environment may have waiting to be delivered. Past it, ingest answers 503 with Retry-After. 1073741824 is 1 GiB.",
+  },
 ];
 
 type Props = {

apps/webapp/app/env.server.ts

@@ -2032,6 +2032,16 @@ const EnvironmentSchema = z
     WEBHOOK_INGRESS_BODY_SIZE_LIMIT_MB: z.coerce.number().int().default(1),
     WEBHOOK_INGRESS_RATE_LIMIT_WINDOW: z.string().default("10s"),
     WEBHOOK_INGRESS_RATE_LIMIT_TOKENS: z.coerce.number().int().default(100),
+    // Per-environment ingress limit across all of its endpoints. A request costs one token per 64 KiB
+    // of body, at least one.
+    WEBHOOK_INGRESS_ENV_RATE_LIMIT_TOKENS: z.coerce.number().int().positive().default(3_000),
+    WEBHOOK_INGRESS_ENV_RATE_LIMIT_WINDOW: z.string().default("10s"),
+    // The most webhook job payload bytes one environment may have queued or in flight.
+    WEBHOOK_QUEUED_PAYLOAD_BYTES_LIMIT: z.coerce
+      .number()
+      .int()
+      .nonnegative()
+      .default(1024 * 1024 * 1024),
     WEBHOOK_INGRESS_IP_RATE_LIMIT_WINDOW: z.string().default("10s"),
     WEBHOOK_INGRESS_IP_RATE_LIMIT_TOKENS: z.coerce.number().int().default(300),
     WEBHOOK_FRONT_GATE_DEFAULT_TTL_SECONDS: z.coerce

apps/webapp/app/presenters/v3/LimitsPresenter.server.ts

@@ -81,6 +81,7 @@ export type LimitsResult = {
     webhookConcurrency: QuotaInfo | null;
     webhookDeliveryRetentionDays: QuotaInfo | null;
     webhookEndpointRateLimit: QuotaInfo | null;
+    webhookEnvironmentRateLimit: QuotaInfo | null;
   };
   features: {
     hasStagingEnvironment: FeatureInfo;
@@ -436,6 +437,17 @@ export class LimitsPresenter extends BasePresenter {
             ? { ...quota, unit: `per ${webhookLimits.limits.endpointRateLimitWindowSeconds}s` }
             : null;
         })(),
+        webhookEnvironmentRateLimit: (() => {
+          const quota = webhookQuota(
+            "environmentRateLimitTokens",
+            "Webhook environment rate limit",
+            `Tokens all of an environment's webhook endpoints accept every ${webhookLimits.limits.environmentRateLimitWindowSeconds.toLocaleString()} seconds. A request costs one token per started 64 KiB of body. More get a 429 with Retry-After.`,
+            0
+          );
+          return quota
+            ? { ...quota, unit: `per ${webhookLimits.limits.environmentRateLimitWindowSeconds}s` }
+            : null;
+        })(),
       },
       features: {
         hasStagingEnvironment: {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx

@@ -552,6 +552,7 @@ function QuotasSection({
   if (quotas.webhookConcurrency) quotaRows.push(quotas.webhookConcurrency);
   if (quotas.webhookDeliveryRetentionDays) quotaRows.push(quotas.webhookDeliveryRetentionDays);
   if (quotas.webhookEndpointRateLimit) quotaRows.push(quotas.webhookEndpointRateLimit);
+  if (quotas.webhookEnvironmentRateLimit) quotaRows.push(quotas.webhookEnvironmentRateLimit);
 
   const showSelfServe = useShowSelfServe();
 
@@ -612,7 +613,8 @@ function QuotaRow({
     quota.name === "Charts per dashboard" ||
     quota.name === "Webhook waiters per endpoint" ||
     quota.name === "Webhook processing concurrency" ||
-    quota.name === "Webhook endpoint rate limit";
+    quota.name === "Webhook endpoint rate limit" ||
+    quota.name === "Webhook environment rate limit";
   const isRetentionQuota = isDurationQuota || isPerItemQuota;
   const isQueueSizeQuota = quota.name === "Max queued runs";
   const hideCurrentUsage = isRetentionQuota || isQueueSizeQuota;

apps/webapp/app/routes/admin.back-office.orgs.$orgId.tsx

@@ -26,8 +26,8 @@ import {
   MaxProjectsSection,
 } from "~/components/admin/backOffice/MaxProjectsSection";
 import { handleMaxProjectsAction } from "~/components/admin/backOffice/MaxProjectsSection.server";
+import { WEBHOOK_LIMITS_INTENT } from "~/components/admin/backOffice/WebhookLimitsForm";
 import {
-  WEBHOOK_LIMITS_INTENT,
   WEBHOOK_LIMITS_SAVED_VALUE,
   WebhookLimitsSection,
 } from "~/components/admin/backOffice/WebhookLimitsSection";

apps/webapp/app/routes/webhooks.v1.ingest.$opaqueId.ts

@@ -37,7 +37,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return toWebhookHttpResponse(webhookHttpResponseFor(result));
   }
 
-  const body = await readWebhookIngressRequest(request);
+  const body = await readWebhookIngressRequest(request, admitted);
   if (body instanceof Response) return body;
   const { rawBytes, headers } = body;
 

apps/webapp/app/routes/webhooks.v1.ingest.$opaqueId_.w.$waiterToken.ts

@@ -18,7 +18,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Method not allowed" }, { status: 405, headers: { Allow: "POST" } });
   }
 
-  const body = await readWebhookIngressRequest(request);
+  const body = await readWebhookIngressRequest(request, admitted);
   if (body instanceof Response) return body;
 
   const result = await webhookEngine.ingestWaiter({